diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..fb29e8c01655d95af4c01050d389308dcc3189d1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +assets/teaser_v2_00.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ORIGINAL_README.md b/ORIGINAL_README.md new file mode 100644 index 0000000000000000000000000000000000000000..01b610677ee8b6827dd5b16e60a4524eae576472 --- /dev/null +++ b/ORIGINAL_README.md @@ -0,0 +1,314 @@ +# Relit-LiVE: Relight Video by Jointly Learning Environment Video + +
+Weiqing Xiao1,* Hong Li2,3,* Xiuyu Yang4,* Houyuan Chen5 Wenyi Li6 Tianqi Liu7 Shaocong Xu2 Chongjie Ye8 Hao Zhao4,2,† Beibei Wang1,† +
+ +
+1Nanjing University  +2BAAI  +3Beihang University  +4Tsinghua University  +5HKUST  +6UCAS  +7HUST  +8CUHK-Shenzhen
+*Equal contribution.  +†Corresponding authors. +
+ +

+ Nanjing University  + BAAI  + Beihang University  + Tsinghua University +

+ +

+ + Paper + + + Website + + + HuggingFace Model + + + License + +

+ +This repo contains the official code of our paper: [Relit-LiVE: Relight Video by Jointly Learning Environment Video](https://arxiv.org/pdf/2605.06658). + +## πŸ“Š Overview + +

+ Overview +

+ +We present **Relit-LiVE**, a novel video relighting framework that produces physically consistent and temporally stable results without needing prior knowledge of camera pose. This is achieved by jointly generating relighting videos and environment videos. Additionally, by integrating real-world lighting effects with intrinsic constraints, the relighting videos demonstrate remarkable physical plausibility, showcasing realistic reflections and shadows. + +## ✨ News + +- May 8, 2026: Release project page and infer pipeline. + +## πŸ“ Check list + +- [x] Release `the arxiv` and `project page`. +- [x] Release `inference code` and `model checkpoints`. +- [x] Release `gradio code` and `full inference pipeline (inverse-forward)`. +- [ ] Release `training code` and `data pipeline`. +- [ ] Release `training dataset`. + +--- + +## πŸ› οΈ Installation + +### Minimum requirements + +- Python 3.10 +- NVIDIA GPU, with at least 24 GB VRAM recommended +- CUDA 12.4 or a compatible version +- Model weights prepared under `checkpoints/` and `models/Wan-AI/Wan2.1-T2V-1.3B/` + +Recommended environment: + +- Ubuntu 20.04 or newer +- Single-GPU CUDA inference setup + +### Conda environment + +```bash +conda create -n diffsynth python=3.10 +conda activate diffsynth +pip install -e . +pip install -U deepspeed +pip install transformers==4.50.0 +pip install gradio==6.14.0 +``` + +### Optional for full inference pipeline + +The [cosmos-transfer1-diffusion-renderer](https://github.com/nv-tlabs/cosmos-transfer1-diffusion-renderer) repository is essential for *full pipeline inference*. Install the conda environment named **cosmos-predict1** following the instructions in its `README.md`. + +```bash +cd third_party +git clone https://github.com/nv-tlabs/cosmos-transfer1-diffusion-renderer.git +... +``` + +## πŸ“¦ Checkpoints + +Download the **Relit-LiVE** checkpoints from HuggingFace and place them under `checkpoints/`. + +| Checkpoint | Resolution | Frames | Download | +| :--- | :---: | :---: | :---: | +| `model_frame25_480_832.ckpt` | 480 Γ— 832 | 8n+1, n∈{0,1,2,3} β†’ 1/9/17/25 | [πŸ€— Download](https://huggingface.co/weiqingXiao/Relit-LiVE) | +| `model_frame57_480_832.ckpt` | 480 Γ— 832 | 8n+1, n∈{0,…,7} β†’ 1/9/…/57 | [πŸ€— Download](https://huggingface.co/weiqingXiao/Relit-LiVE) | +| `model_frame1_1024_1472.ckpt` | 1024 Γ— 1472 | 1 (image) | [πŸ€— Download](https://huggingface.co/weiqingXiao/Relit-LiVE) | + +In addition, inference loads the Wan2.1 base model from `models/Wan-AI/Wan2.1-T2V-1.3B/`. Make sure all weights are in place before running inference. + +**If you want to reproduce the MIT metrics reported in the paper, you should load the `model_frame57_480_832.ckpt` and perform single-frame inference directly on the test set.** + +**(Optional for full inference pipeline)** Download the **cosmos-transfer1-diffusion-renderer** checkpoints from HuggingFace and place them under `third_party/cosmos-transfer1-diffusion-renderer/checkpoints/` following the instructions in its `README.md`. + +## πŸš€ Inference + +By default, generated results are written to `inference_output/`. + +### Basic 25-frame relighting + +```bash +python relit_inference.py \ + --dataset_path datasets/demos \ + --ckpt_path checkpoints/model_frame25_480_832.ckpt \ + --output_dir inference_output \ + --cfg_scale 1.0 \ + --height 480 \ + --width 832 \ + --num_frames 25 \ + --padding_resolution \ + --use_ref_image \ + --env_map_path datasets/envs/Pink_Sunrise \ + --frame_interval 1 \ + --num_inference_steps 50 \ + --quality 10 +``` + +### 25-frame rotating-light relighting + +```bash +python relit_inference.py \ + --dataset_path datasets/demos \ + --ckpt_path checkpoints/model_frame25_480_832.ckpt \ + --output_dir inference_output \ + --cfg_scale 1.0 \ + --height 480 \ + --width 832 \ + --num_frames 25 \ + --padding_resolution \ + --use_ref_image \ + --env_map_path datasets/envs/Pink_Sunrise \ + --frame_interval 1 \ + --num_inference_steps 50 \ + --use_rotate_light \ + --quality 10 +``` + +### Fixed-frame relighting with width-axis light rotation + +```bash +python relit_inference.py \ + --dataset_path datasets/demos \ + --ckpt_path checkpoints/model_frame25_480_832.ckpt \ + --output_dir inference_output \ + --cfg_scale 1.0 \ + --height 480 \ + --width 832 \ + --num_frames 25 \ + --padding_resolution \ + --use_ref_image \ + --env_map_path datasets/envs/Pink_Sunrise \ + --frame_interval 1 \ + --num_inference_steps 50 \ + --use_fixed_frame_and_w_rotate_light \ + --quality 10 +``` + +### Fixed-frame relighting with height-axis light rotation + +```bash +python relit_inference.py \ + --dataset_path datasets/demos \ + --ckpt_path checkpoints/model_frame25_480_832.ckpt \ + --output_dir inference_output \ + --cfg_scale 1.0 \ + --height 480 \ + --width 832 \ + --num_frames 25 \ + --padding_resolution \ + --use_ref_image \ + --env_map_path datasets/envs/Pink_Sunrise \ + --frame_interval 1 \ + --num_inference_steps 50 \ + --use_fixed_frame_and_h_rotate_light \ + --quality 10 +``` + +### 57-frame video relighting + +```bash +python relit_inference.py \ + --dataset_path datasets/demos \ + --ckpt_path checkpoints/model_frame57_480_832.ckpt \ + --output_dir inference_output \ + --cfg_scale 1.0 \ + --height 480 \ + --width 832 \ + --num_frames 57 \ + --padding_resolution \ + --use_ref_image \ + --env_map_path datasets/envs/Pink_Sunrise \ + --frame_interval 1 \ + --num_inference_steps 50 \ + --quality 10 +``` + +### Single-frame high-resolution relighting + +```bash +python relit_inference.py \ + --dataset_path datasets/demos \ + --ckpt_path checkpoints/model_frame1_1024_1472.ckpt \ + --output_dir inference_output \ + --cfg_scale 1.0 \ + --height 1024 \ + --width 1472 \ + --num_frames 1 \ + --padding_resolution \ + --use_ref_image \ + --env_map_path datasets/envs/Pink_Sunrise \ + --frame_interval 1 \ + --num_inference_steps 50 \ + --quality 10 +``` + +## πŸ“‹ Argument reference + +The following arguments are defined in `parse_args()` inside `relit_inference.py`. + +| Argument | Type | Default | Description | +| --- | --- | --- | --- | +| `--dataset_path` | str | `./example_test_data` | Input dataset directory. The examples above use `datasets/demos`. | +| `--env_map_path` | str | `None` | External environment map directory. If not provided, the script reads lighting data from each sample. | +| `--use_ref_image` | flag | `False` | Enable the reference-image branch. | +| `--use_muti_ref_image` | flag | `False` | Enable multi-reference-image mode. The argument name follows the current code spelling. | +| `--ref_image_path_with_idddx` | str | `None` | Template path for external reference images. The script replaces `idddx` with the sample index. | +| `--full_resolution` | flag | `False` | Use the full-resolution input pipeline. | +| `--padding_resolution` | flag | `False` | Use a padding-based resize strategy to reduce aggressive cropping. | +| `--dataset_type` | str | `relit-live` | Dataset format. The default matches the Relit-LiVE directory structure in this repository. | +| `--drop_mr` | flag | `False` | Ignore metallic and roughness conditioning. | +| `--use_rotate_light` | flag | `False` | Enable dynamic light rotation mode. | +| `--use_fixed_frame_and_w_rotate_light` | flag | `False` | Keep the first frame fixed and rotate lighting along the environment-map width axis. | +| `--use_fixed_frame_and_h_rotate_light` | flag | `False` | Keep the first frame fixed and rotate lighting along the environment-map height axis. | +| `--h_rotate_light` | int | `0` | Apply vertical environment-map rotation to each frame, in degrees. | +| `--w_rotate_light` | int | `0` | Apply horizontal environment-map rotation to each frame, in pixels. | +| `--num_frames` | int | `81` | Number of output frames. When set to `1`, the script saves a png; otherwise it saves an mp4. | +| `--num_inference_steps` | int | `50` | Number of denoising inference steps. | +| `--frame_interval` | int | `1` | Sampling interval when reading the input video or image sequence. | +| `--height` | int | `480` | Output height. | +| `--width` | int | `832` | Output width. | +| `--ckpt_path` | str | `None` | Path to the checkpoint to load. | +| `--output_dir` | str | `./results` | Default output directory. | +| `--output_path` | str | `None` | Explicit output file path. Only `.mp4` and `.png` are supported. | +| `--dataloader_num_workers` | int | `1` | Number of DataLoader workers. | +| `--cfg_scale` | float | `5.0` | Classifier-free guidance scale. | +| `--wo_ref_weight` | float | `0.0` | Weight for the branch without reference-image conditioning. | +| `--quality` | int | `5` | Video quality value passed to `imageio` when saving mp4 files. | + +### Notes + +- Output filenames automatically include parts of the checkpoint name, sequence name, resolution, reference-image mode, environment lighting information, inference steps, frame count, and `cfg_scale`. +- When `--num_frames 1` is used, the script writes a png. When `--num_frames > 1`, it writes an mp4. + +## πŸš€ Full inference pipeline (gradio) + +Please make sure you have the following items ready: + +1. conda environment named **diffsynth**. +2. conda environment named **cosmos-predict1**. +3. `./checkpoints/*.ckpt`. +4. `./third_party/cosmos-transfer1-diffusion-renderer`. +5. `./third_party/cosmos-transfer1-diffusion-renderer/checkpoints/Cosmos-Tokenize1-CV8x8x8-720p` and `./third_party/cosmos-transfer1-diffusion-renderer/checkpoints/Diffusion_Renderer_Inverse_Cosmos_7B`. + +Then, + +```bash +conda activate diffsynth +python run_full_inference_gradio.py +``` + +## πŸ“Œ Future plans + +This project will be continuously maintained. We welcome users to try it out and share their feedback (15770575681@163.com). + +The current plan includes a model version **specifically designed for portraits** and another that is better suited for **handling motion** (including camera and scene dynamics). + +## 🀝 Citation + +If you find this repository helpful, please consider citing our paper: + +```bibtex +@article{xiao2026relit, + title={Relit-LiVE: Relight Video by Jointly Learning Environment Video}, + author={Xiao, Weiqing and Li, Hong and Yang, Xiuyu and Chen, Houyuan and Li, Wenyi and Liu, Tianqi and Xu, Shaocong and Ye, Chongjie and Zhao, Hao and Wang, Beibei}, + journal={arXiv preprint arXiv:2605.06658}, + year={2026} +} +``` + +## πŸ“ Acknowledgements + +Code is built on [DiffSynth-Studio](https://github.com/modelscope/DiffSynth-Studio) and [diffusion-renderer](https://github.com/nv-tlabs/cosmos-transfer1-diffusion-renderer). Thanks all the authors for their excellent contributions! \ No newline at end of file diff --git a/assets/badge-website.svg b/assets/badge-website.svg new file mode 100644 index 0000000000000000000000000000000000000000..7231a99946f500887b2f8197b64cd7e30f995742 --- /dev/null +++ b/assets/badge-website.svg @@ -0,0 +1,129 @@ + + + + + project: website + + + + + + + + + + + + + + + + Project + + + Website + + + + diff --git a/assets/logos/baai-logo.png b/assets/logos/baai-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..448d32c9dd863990c926e909e714087bbfe94b28 Binary files /dev/null and b/assets/logos/baai-logo.png differ diff --git a/assets/logos/buaa-logo.png b/assets/logos/buaa-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..901009699cbcae46629e06718977bb3078e81417 Binary files /dev/null and b/assets/logos/buaa-logo.png differ diff --git a/assets/logos/nju-logo.png b/assets/logos/nju-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..9ca294e70c9761d4496360ddd42c05ca4e300d9d Binary files /dev/null and b/assets/logos/nju-logo.png differ diff --git a/assets/logos/tsinghua-logo.png b/assets/logos/tsinghua-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..cb1a8b7f36d4e3b0911b75599b00b78088a3021e Binary files /dev/null and b/assets/logos/tsinghua-logo.png differ diff --git a/assets/teaser_v2_00.jpg b/assets/teaser_v2_00.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3a0061b6f92d33c529c305bf16a60c699b5d4492 --- /dev/null +++ b/assets/teaser_v2_00.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2c955a140c5956b73c79c3978a416bb80a0aeb7b4dc8579ad0435d3885ded0a +size 212175 diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0000.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0000.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b3618230dd205cd97e6587278ada42651b29e97 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0000.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0001.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..43e32be0fecc2f69fdc8be2287ab03d49fbd0346 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0001.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0002.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0002.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bbd98c9490ceab0eaaeaa0f8c407bbad2bf6bc85 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0002.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0003.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0003.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8ae5b667978a19f8122fe8e195c22621b68c79fc Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0003.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0004.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0004.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51f1722fdae68c8304bc35e5ae197badc416dbbf Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0004.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0005.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0005.jpg new file mode 100644 index 0000000000000000000000000000000000000000..302aa8a9d80d44891ab5e38f1711f0e998ef4e34 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0005.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0006.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0006.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e3924f60b562e307de4a3fd602249b11f8747f70 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0006.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0007.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0007.jpg new file mode 100644 index 0000000000000000000000000000000000000000..243fb424652709c51ca5410344dc8c0af2fc319f Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0007.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0008.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0008.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a43c244f6d86e903b24281452785bd021ae82fca Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0008.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0009.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0009.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cd6e96ceca023e6e979c420c9951be2d2a202a53 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0009.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0010.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0010.jpg new file mode 100644 index 0000000000000000000000000000000000000000..874f0849dd62f5dc42b3ae99cdbbe1431012cdcd Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0010.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0011.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0011.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2754f2f090b6867f0273d901e5e57638115b941a Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0011.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0012.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0012.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3a0a5001b024cebabb476c11a6932b087b83c19b Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0012.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0013.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0013.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8815ebab78148a623ee480a9c621d860b4f9d215 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0013.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0014.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0014.jpg new file mode 100644 index 0000000000000000000000000000000000000000..39ecdaa007271057dc15ee5479fbf5899a616cda Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0014.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0015.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0015.jpg new file mode 100644 index 0000000000000000000000000000000000000000..201d5d6986dbbb3d27ed282683f42bf3d11c08ec Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0015.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0016.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0016.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e8967f108adb9c2c7722a1e2f98e8789a1f8d658 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0016.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0017.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0017.jpg new file mode 100644 index 0000000000000000000000000000000000000000..93951d15c7413eedc491d2306a04e69ef5bdc8fe Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0017.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0018.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0018.jpg new file mode 100644 index 0000000000000000000000000000000000000000..6b48315bbe1d345d634257fac1820d9baabc0e31 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0018.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0019.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0019.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b7dc0372c5ae1207ac1a0a47eaba438a66e4b675 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0019.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0020.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0020.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c55f2a4eab8c677182b74b7e1cadb332e5050dab Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0020.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0021.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0021.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f9422be785250347033e193eab0a17735b2a2893 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0021.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0022.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0022.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f6873145d065e123437d9b838a126a7707a5cde8 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0022.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0023.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0023.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7a9bb28d2dc4db007798240cc0df8018dc0c30f7 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0023.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0024.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0024.jpg new file mode 100644 index 0000000000000000000000000000000000000000..566b181a78fa33e837c5335c789db0cecd436e6a Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0024.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0025.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0025.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f1aac072ca41a22f2eb9aa87583e8590c62811b2 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0025.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0026.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0026.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1908686a0293586319fc9b44d17d0fedd4e45cd0 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0026.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0027.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0027.jpg new file mode 100644 index 0000000000000000000000000000000000000000..571a37dba42f7c1ef7b390d74d4fb69c660a6620 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0027.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0028.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0028.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0b9287e07db48b4d71086b2f12769399eb6af9b0 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0028.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0029.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0029.jpg new file mode 100644 index 0000000000000000000000000000000000000000..df72b6a19ce9b1b3ca6f8685d338b32f457ca6f7 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0029.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0030.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0030.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f68cf9701bc713188fb73d235be7dde3a5eeee97 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0030.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0031.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0031.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8d7d6b43ae353011a33287b9efba808c986863f6 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0031.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0032.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0032.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0488619f9d386ceae57120ca5ac654a257996b97 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0032.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0033.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0033.jpg new file mode 100644 index 0000000000000000000000000000000000000000..49bfb1b1487c137f2d6fad8b23859fca866e54a2 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0033.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0034.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0034.jpg new file mode 100644 index 0000000000000000000000000000000000000000..20ab86376074ea4ed85cbf7de04b90190b4e0b6a Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0034.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0035.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0035.jpg new file mode 100644 index 0000000000000000000000000000000000000000..999576ccdcd909a05701cd5194e7823dd4da945f Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0035.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0036.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0036.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ffe32b57b53354b785486b0f25353a407a307502 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0036.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0037.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0037.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7efccc870946c2d6649edd3ce9e91ba8aa76a5f5 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0037.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0038.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0038.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1de4b3dfb02daca3f8dd1fabbfdc211691e4fb06 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0038.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0039.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0039.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8cf1db292f8b61ac1436f7ba4e68ebcf31a4f045 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0039.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0040.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0040.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bb16212633147a21181e74b353b14004b6dd4d7d Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0040.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0041.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0041.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dfd6270e4521eef9f4b7f432c365547ed0956a12 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0041.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0042.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0042.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ec921444ff4603b8c3d3a8ec44112298e962d397 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0042.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0043.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0043.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ec28b8f7691eefe7050e28a78497dcd71e7de1ae Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0043.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0044.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0044.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0569a6888ea9dc9ce555409b17b4fb680b9383e1 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0044.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0045.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0045.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b93d8ad0642a77f2a5f0028d1743861d5da9c983 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0045.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0046.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0046.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f0e65554684c5088adcbb2d1ef482f1bea88b93 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0046.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0047.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0047.jpg new file mode 100644 index 0000000000000000000000000000000000000000..043f1b7d960ad3e89e80bfec2b54ea2955185857 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0047.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0048.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0048.jpg new file mode 100644 index 0000000000000000000000000000000000000000..89b1db0890f4f462f0b1bb12f160642212af7634 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0048.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0049.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0049.jpg new file mode 100644 index 0000000000000000000000000000000000000000..110b041f102545fad69a7bf127af443de8ba37d6 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0049.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0050.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0050.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d79edbc2f2e626cc177079c36f0b4c58d1e5247c Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0050.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0051.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0051.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4588c6ba31b4b3b10f63f9125428f03908011073 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0051.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0052.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0052.jpg new file mode 100644 index 0000000000000000000000000000000000000000..023d157a7440fa5a0c332d8a50ea3a5a04a23ab5 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0052.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0053.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0053.jpg new file mode 100644 index 0000000000000000000000000000000000000000..36a5ff7a5e060604a6b96e517f1691e64359e0ed Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0053.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0054.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0054.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f747483e0dfa02a3bb11053ff74bc190c096c4e9 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0054.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0055.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0055.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c87527a0f32c36092a7091cb60c3cdc724c5279a Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0055.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Base Color/frame_0056.jpg b/datasets/demos/human_movies_human_15.0/Base Color/frame_0056.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0ef00c46f06aa33963e1002fd8d6134b0753f776 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Base Color/frame_0056.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Metallic/frame_0000.jpg b/datasets/demos/human_movies_human_15.0/Metallic/frame_0000.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8d13dda812fe7868c19d23fbdfb5292fb9d0d9cc Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Metallic/frame_0000.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Metallic/frame_0001.jpg b/datasets/demos/human_movies_human_15.0/Metallic/frame_0001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f242024584be5b9f09f24190ce4fc0fa71ebadbe Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Metallic/frame_0001.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Metallic/frame_0002.jpg b/datasets/demos/human_movies_human_15.0/Metallic/frame_0002.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d5360cc681c6f0f5886a8923a4ec366340b5ce4c Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Metallic/frame_0002.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Metallic/frame_0003.jpg b/datasets/demos/human_movies_human_15.0/Metallic/frame_0003.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c6a7c9d313d4f9ea3d5af67fd3a1dac409c88871 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Metallic/frame_0003.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Metallic/frame_0004.jpg b/datasets/demos/human_movies_human_15.0/Metallic/frame_0004.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8a31af1a4f25f880f0215ea3431f750204538483 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Metallic/frame_0004.jpg differ diff --git a/datasets/demos/human_movies_human_15.0/Metallic/frame_0005.jpg b/datasets/demos/human_movies_human_15.0/Metallic/frame_0005.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4ea77889f1ec32069b0806ada565b2ea956ad1e9 Binary files /dev/null and b/datasets/demos/human_movies_human_15.0/Metallic/frame_0005.jpg differ diff --git a/relit_inference.py b/relit_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..34c454d7db08dd6e2b61bc86750e9ed5050893f8 --- /dev/null +++ b/relit_inference.py @@ -0,0 +1,839 @@ + +from diffsynth import ModelManager, WanVideoRelitlivePipeline +from torchvision.transforms import v2 +from einops import rearrange +from PIL import Image +from tqdm import tqdm +from natsort import natsorted +from typing import List + +import torch, os, imageio, argparse, torchvision, pdb, cv2, pathlib, glob, pyexr +import torch.nn as nn +import numpy as np + +def stitch_frames(video_frames, rows=2, cols=5): + num_frames = len(video_frames[0]) + assert all(len(frames) == num_frames for frames in video_frames), "All video frames must have the same number of frames." + + frame_width, frame_height = video_frames[0][0].size + stitched_images = [] + for frame_idx in range(num_frames): + stitched_image = Image.new('RGB', (cols * frame_width, rows * frame_height)) + for i in range(rows): + for j in range(cols): + video_idx = i * cols + j + if video_idx < len(video_frames): + img = video_frames[video_idx][frame_idx] + x = j * frame_width + y = i * frame_height + stitched_image.paste(img, (x, y)) + stitched_images.append(stitched_image) + + return stitched_images + +def save_video(frames, save_path, fps, quality=9, ffmpeg_params=None): + writer = imageio.get_writer(save_path, fps=fps, quality=quality, ffmpeg_params=ffmpeg_params) + for frame in tqdm(frames, desc="Saving video"): + frame = np.array(frame) + writer.append_data(frame) + writer.close() + +def save_frames(frames, save_path): + os.makedirs(save_path, exist_ok=True) + for i, frame in enumerate(tqdm(frames, desc="Saving images")): + frame.save(os.path.join(save_path, f"{i}.png")) + +def rotate_panorama_around_horizontal_axis( + panorama_img, + pitch_angle=10 +): + if isinstance(panorama_img, Image.Image): + panorama = np.array(panorama_img.convert('RGB')) + elif isinstance(panorama_img, np.ndarray): + panorama = panorama_img.copy() + if len(panorama.shape) != 3 or panorama.shape[2] != 3: + raise ValueError("The NumPy array for the panoramic image must be in RGB format with dimensions (H, W, 3).") + else: + raise TypeError("The panoramic image must be a PIL.Image or a numpy.ndarray.") + + pan_h, pan_w = panorama.shape[:2] + + pitch_rad = np.radians(pitch_angle) + + lon = np.linspace(0, 2 * np.pi, pan_w) + lat = np.linspace(-np.pi/2, np.pi/2, pan_h) + lon_grid, lat_grid = np.meshgrid(lon, lat) + + x = np.cos(lat_grid) * np.sin(lon_grid) + y = np.sin(lat_grid) + z = np.cos(lat_grid) * np.cos(lon_grid) + coords = np.stack([x, y, z], axis=-1) + + R_pitch = np.array([ + [1, 0, 0], + [0, np.cos(pitch_rad), -np.sin(pitch_rad)], + [0, np.sin(pitch_rad), np.cos(pitch_rad)] + ]) + coords_rot = np.dot(coords, R_pitch.T) + + lon_new = np.arctan2(coords_rot[..., 0], coords_rot[..., 2]) + lat_new = np.arcsin(np.clip(coords_rot[..., 1], -1, 1)) + + u_new = (lon_new / (2 * np.pi) + 0.5) * pan_w + v_new = (lat_new / np.pi + 0.5) * pan_h + + u_new = u_new.astype(np.float32) + v_new = v_new.astype(np.float32) + + new_panorama = cv2.remap( + panorama, + u_new, v_new, + interpolation=cv2.INTER_CUBIC, + borderMode=cv2.BORDER_WRAP + ) + + new_panorama = np.hstack([new_panorama[:, pan_w//2:, ...], new_panorama[:, :pan_w//2, ...]]) + return new_panorama + +class PBRVideo_img_Dataset(torch.utils.data.Dataset): + def __init__(self, base_path, max_num_frames=81, frame_interval=1, num_frames=81, height=480, width=832, env_map_path=None, dataset_type='relit-live', \ + use_ref_image=False, full_resolution=False, padding_resolution=False, drop_mr=False, args=None): + + self.dataset_type = dataset_type + self.base_path = base_path + self.args = args + self.num_frames = num_frames + + p = pathlib.Path(base_path) + if not p.is_dir(): + raise NotADirectoryError(f"{base_path}' is not a valid dir") + + if self.dataset_type == "relit-live": + self.path: List[pathlib.Path] = natsorted([item for item in p.iterdir() if item.is_dir()]) + self.frame_interval = frame_interval if self.num_frames!=1 else 0 + + print(f'============= Load {len(self.path)}seqs from {base_path} =============') + + self.max_num_frames = max_num_frames + self.num_frames = num_frames + self.height = height + self.width = width + self.frame_process = v2.Compose([ + v2.CenterCrop(size=(height, width)), + v2.ToTensor(), + v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + ]) + + self.env_map_path = env_map_path + self.use_ref_image = use_ref_image + self.full_resolution = full_resolution + self.padding_resolution = padding_resolution + self.drop_mr = drop_mr + + def crop_and_resize(self, image, std_shape=None, env_reshape=False, align=None): + if env_reshape: + image = torchvision.transforms.functional.resize( + image, + (self.height, self.width), + interpolation=torchvision.transforms.InterpolationMode.BILINEAR + ) + else: + if std_shape is not None: + height, width = std_shape + else: + width, height = image.size + if self.padding_resolution: + scale = min(self.width / width, self.height / height) + else: + scale = max(self.width / width, self.height / height) + if align is not None: + if align == 'width': + scale = self.width / width + elif align == 'height': + scale = self.height / height + image = torchvision.transforms.functional.resize( + image, + (round(height*scale), round(width*scale)), + interpolation=torchvision.transforms.InterpolationMode.BILINEAR + ) + return image + + def load_frames_using_imageio(self, file_path, start_frame_id, interval, num_frames, frame_process, std_shape=None, env_reshape=False, return_shape=False): + reader = imageio.get_reader(file_path) + image_num = reader.count_frames() + + frames = [] + for frame_id in range(num_frames): + sample_list = list(range(image_num)) + list(range(image_num-2, 0, -1)) + sample_id = sample_list[((start_frame_id + frame_id * interval) % len(sample_list))] + frame = reader.get_data(sample_id) + img_shape = frame.shape[:2] + frame = Image.fromarray(frame) + frame = self.crop_and_resize(frame, std_shape, env_reshape) + frame = frame_process(frame) + frames.append(frame) + reader.close() + + frames = torch.stack(frames, dim=0) + frames = rearrange(frames, "T C H W -> C T H W") + + if return_shape: + return frames, img_shape + return frames + + def load_frames_using_imageio_from_imgdir(self, file_path, start_frame_id, interval, num_frames, frame_process, divided_max=False, std_shape=None, env_reshape=False, return_shape=False): + image_paths = glob.glob(os.path.join(file_path, '*.png')) + glob.glob(os.path.join(file_path, '*.jpg')) + glob.glob(os.path.join(file_path, '*.exr')) + if len(image_paths) == 0: + return None + + sorted_image_paths = natsorted(image_paths) + image_num = len(sorted_image_paths) + + frames = [] + for frame_id in range(num_frames): + sample_list = list(range(image_num)) + list(range(image_num-2, 0, -1)) + sample_id = sample_list[((start_frame_id + frame_id * interval) % len(sample_list))] + + if ".exr" in sorted_image_paths[sample_id]: + frame = pyexr.open(sorted_image_paths[sample_id]).get() + else: + frame = imageio.imread(sorted_image_paths[sample_id]) + + if frame.shape[-1] == 4: + frame = frame[:,:,:3] + elif frame.shape[-1] == 1: + frame = np.repeat(frame, 3, axis=-1) + + img_shape = frame.shape[:2] + if divided_max: + bg_mask = (frame > 1000.0) + if len(frame[bg_mask]) > 0: + frame_max = float(np.percentile(frame, 99)) + frame = np.clip(frame / frame_max, 0, 1) + else: + frame = frame / frame.max() + + if frame.min() < 0: + frame = frame * 0.5 + 0.5 + + if frame.dtype != 'uint8': + frame = (frame * 255).astype(np.uint8) + + frame = Image.fromarray(frame) + frame = self.crop_and_resize(frame, std_shape, env_reshape) + frame = frame_process(frame) + frames.append(frame) + + frames = torch.stack(frames, dim=0) + frames = rearrange(frames, "T C H W -> C T H W") + + if return_shape: + return frames, img_shape + return frames + + def load_frames_using_imageio_from_imgpath(self, file_path, start_frame_id, interval, num_frames, frame_process, divided_max=False, std_shape=None, env_reshape=False, return_shape=False, align=None): + image_paths = [file_path] + if len(image_paths) == 0: + return None + + sorted_image_paths = natsorted(image_paths) + image_num = len(sorted_image_paths) + + frames = [] + for frame_id in range(num_frames): + sample_list = list(range(image_num)) + list(range(image_num-2, 0, -1)) + sample_id = sample_list[((start_frame_id + frame_id * interval) % len(sample_list))] + + if ".exr" in sorted_image_paths[sample_id]: + frame = pyexr.open(sorted_image_paths[sample_id]).get() + else: + frame = imageio.imread(sorted_image_paths[sample_id]) + + if frame.shape[-1] == 4: + frame = frame[:,:,:3] + elif frame.shape[-1] == 1: + frame = np.repeat(frame, 3, axis=-1) + + img_shape = frame.shape[:2] + + if divided_max: + bg_mask = (frame > 1000.0) + if len(frame[bg_mask]) > 0: + frame_max = float(np.percentile(frame, 99)) + frame = np.clip(frame / frame_max, 0, 1) + else: + frame = frame / frame.max() + + if frame.min() < 0: + frame = frame * 0.5 + 0.5 + + if frame.dtype != 'uint8': + frame = (frame * 255).astype(np.uint8) + + frame = Image.fromarray(frame) + frame = self.crop_and_resize(frame, std_shape, env_reshape, align=align) + frame = frame_process(frame) + frames.append(frame) + frames = torch.stack(frames, dim=0) + frames = rearrange(frames, "T C H W -> C T H W") + + if return_shape: + return frames, img_shape + return frames + + def __getitem__(self, index): + data_id = index % len(self.path) + dir_path = self.path[data_id] + + RGB_path = os.path.join(dir_path, "images_4") + basecolor_path = os.path.join(dir_path, "Base Color") + depth_path = os.path.join(dir_path, "depth") + metallic_path = os.path.join(dir_path, "Metallic") + normal_path = os.path.join(dir_path, "normal") + roughness_path = os.path.join(dir_path, "Roughness") + + fixed_env = True + if self.env_map_path is not None: + ldr_path = os.path.join(self.env_map_path, "ldr_video_fix_first_frame.mp4") + hdr_log_path = os.path.join(self.env_map_path, "hdr_log_video_fix_first_frame.mp4") + env_dir_path = os.path.join(self.env_map_path, "env_dir_video_fix_first_frame.mp4") + else: + ldr_path = os.path.join(dir_path, "env", "ldr_video_fix_first_frame.mp4") + hdr_log_path = os.path.join(dir_path, "env", "hdr_log_video_fix_first_frame.mp4") + env_dir_path = os.path.join(dir_path, "env", "env_dir_video_fix_first_frame.mp4") + fixed_env = False + + start_frame_id = 0 + + if self.full_resolution: + source = self.load_frames_using_imageio_from_imgdir(RGB_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + + basecolor= self.load_frames_using_imageio_from_imgdir(basecolor_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + depth= self.load_frames_using_imageio_from_imgdir(depth_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, divided_max=True, env_reshape=True) + metallic= self.load_frames_using_imageio_from_imgdir(metallic_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + + normal= self.load_frames_using_imageio_from_imgdir(normal_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + roughness = self.load_frames_using_imageio_from_imgdir(roughness_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + else: + source, std_shape = self.load_frames_using_imageio_from_imgdir(RGB_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, return_shape=True) + + basecolor= self.load_frames_using_imageio_from_imgdir(basecolor_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, std_shape=std_shape) + depth= self.load_frames_using_imageio_from_imgdir(depth_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, divided_max=True, std_shape=std_shape) + metallic= self.load_frames_using_imageio_from_imgdir(metallic_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, std_shape=std_shape) + + normal= self.load_frames_using_imageio_from_imgdir(normal_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, std_shape=std_shape) + roughness = self.load_frames_using_imageio_from_imgdir(roughness_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, std_shape=std_shape) + + if metallic is None: + metallic = source * 0.0 + if roughness is None: + roughness = source * 0.0 + + ldr = self.load_frames_using_imageio(ldr_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + hdr_log = self.load_frames_using_imageio(hdr_log_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + env_dir = self.load_frames_using_imageio(env_dir_path, start_frame_id, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + + if fixed_env: + ldr = ldr[:,0:1].repeat(1, self.num_frames, 1, 1) + hdr_log = hdr_log[:,0:1].repeat(1, self.num_frames, 1, 1) + + if self.drop_mr: + metallic_weight = torch.tensor(0.0) + roughness_weight = torch.tensor(0.0) + else: + metallic_weight = torch.tensor(1.0) + roughness_weight = torch.tensor(1.0) + + env_self_weight = torch.tensor(1.0) + env_cross_weight = torch.tensor(1.0) + + # print(f"haven't use prompt for {RGB_path}") + prompt = "" + + if self.use_ref_image: + + image_paths = glob.glob(os.path.join(basecolor_path, '*.png')) + glob.glob(os.path.join(basecolor_path, '*.jpg')) + glob.glob(os.path.join(basecolor_path, '*.exr')) + max_idx = (len(image_paths)-1) - (self.num_frames-1) * self.frame_interval + if max_idx == 0: + start_frame_id_ref = 0 + else: + start_frame_id_ref = start_frame_id + + ref_RGB_path = RGB_path + if self.full_resolution: + ref_source = self.load_frames_using_imageio_from_imgdir(ref_RGB_path, start_frame_id_ref, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + else: + ref_source = self.load_frames_using_imageio_from_imgdir(ref_RGB_path, start_frame_id_ref, self.frame_interval, self.num_frames, self.frame_process, std_shape=std_shape) + + if self.args.use_fixed_frame_and_w_rotate_light or self.args.use_fixed_frame_and_h_rotate_light or self.num_frames == 1: + ref_idx = 0 + else: + ref_idx = ref_source.shape[1] // 2 # Use the mid frame as the reference image + + ref_image = ref_source[:, ref_idx:ref_idx+1, :, :] + + if self.args.ref_image_path_with_idddx is not None: + ref_image_path_with_idx = self.args.ref_image_path_with_idddx.replace("idddx", f"{index}") + if self.full_resolution: + ref_image = self.load_frames_using_imageio_from_imgpath(ref_image_path_with_idx, self.max_num_frames, start_frame_id_ref, self.frame_interval, self.num_frames, self.frame_process, env_reshape=True) + else: + ref_image = self.load_frames_using_imageio_from_imgpath(ref_image_path_with_idx, self.max_num_frames, start_frame_id_ref, self.frame_interval, self.num_frames, self.frame_process, std_shape=None, align='width') + ref_image = ref_image[:, -1:, :, :] + + data = { + "source_video": source, + "basecolor": basecolor, + "depth": depth, + "metallic":metallic, + "normal": normal, + "roughness":roughness, + 'metallic_weight': metallic_weight, + 'roughness_weight': roughness_weight, + 'env_self_weight': env_self_weight, + 'env_cross_weight': env_cross_weight, + "ldr": ldr, + "hdr_log": hdr_log, + "env_dir": env_dir, + "ref_image": ref_image, + "ref_video": ref_source, + "path": str(dir_path), + "prompt": prompt, + } + else: + data = { + "source_video": source, + "basecolor": basecolor, + "depth": depth, + "metallic":metallic, + "normal": normal, + "roughness":roughness, + 'metallic_weight': metallic_weight, + 'roughness_weight': roughness_weight, + 'env_self_weight': env_self_weight, + 'env_cross_weight': env_cross_weight, + "ldr": ldr, + "hdr_log": hdr_log, + "env_dir": env_dir, + "path": str(dir_path), + "prompt": prompt, + } + + return data + + def __len__(self): + return len(self.path) + +def parse_args(): + parser = argparse.ArgumentParser(description="diffusion renderer Inference") + parser.add_argument( + "--dataset_path", + type=str, + default="./example_test_data", + help="The path of the Dataset.", + ) + parser.add_argument( + "--env_map_path", + type=str, + default=None, + help="env_map_path.", + ) + parser.add_argument( + "--use_ref_image", + default=False, + action="store_true", + help="Whether to use reference image.", + ) + parser.add_argument( + "--use_muti_ref_image", + default=False, + action="store_true", + help="Whether to use multiple reference images.", + ) + parser.add_argument( + "--ref_image_path_with_idddx", + type=str, + default=None, + help="Path to the reference image with index.", + ) + parser.add_argument( + "--full_resolution", + default=False, + action="store_true", + help="Whether to use full_resolution.", + ) + parser.add_argument( + "--padding_resolution", + default=False, + action="store_true", + help="Whether to use padding_resolution.", + ) + parser.add_argument( + "--dataset_type", + type=str, + default="relit-live", + help="Dataset format to load. Use 'relit-live' for the default Relit-LiVE directory layout.", + ) + parser.add_argument( + "--drop_mr", + default=False, + action="store_true", + help="Ignore metallic and roughness inputs by setting their conditioning weights to zero.", + ) + parser.add_argument( + "--use_rotate_light", + default=False, + action="store_true", + help="Enable light rotation mode during inference.", + ) + parser.add_argument( + "--use_fixed_frame_and_w_rotate_light", + default=False, + action="store_true", + help="Repeat the first frame and rotate the environment map along the width axis across frames.", + ) + parser.add_argument( + "--use_fixed_frame_and_h_rotate_light", + default=False, + action="store_true", + help="Repeat the first frame and rotate the environment map along the height axis across frames.", + ) + parser.add_argument( + "--h_rotate_light", + type=int, + default=0, + help="Rotate the environment map vertically by this many degrees for every frame.", + ) + parser.add_argument( + "--w_rotate_light", + type=int, + default=0, + help="Rotate the environment map horizontally by this many pixels for every frame.", + ) + parser.add_argument( + "--num_frames", + type=int, + default=81, + help="Number of frames.", + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=50, + help="Number of denoising steps used during inference.", + ) + parser.add_argument( + "--frame_interval", + type=int, + default=1, + help="Sampling interval between frames read from the input sequence.", + ) + parser.add_argument( + "--height", + type=int, + default=480, + help="Image height.", + ) + parser.add_argument( + "--width", + type=int, + default=832, + help="Image width.", + ) + parser.add_argument( + "--ckpt_path", + type=str, + default=None, + help="Path to the fine-tuned checkpoint to load into the pipeline.", + ) + parser.add_argument( + "--output_dir", + type=str, + default="./results", + help="Path to save the results.", + ) + parser.add_argument( + "--output_path", + type=str, + default=None, + help="Optional explicit output file path. Supports .mp4 and .png.", + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=1, + help="Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process.", + ) + parser.add_argument( + "--cfg_scale", + type=float, + default=5.0, + help="Classifier-free guidance scale.", + ) + parser.add_argument( + "--wo_ref_weight", + type=float, + default=0.0, + help="Weight applied to the branch without reference-image conditioning.", + ) + parser.add_argument( + "--quality", + type=int, + default=5, + help="Video encoding quality passed to imageio when saving mp4 outputs.", + ) + args = parser.parse_args() + return args + +if __name__ == '__main__': + args = parse_args() + + # Load Wan2.1 pre-trained models + model_manager = ModelManager(torch_dtype=torch.bfloat16, device="cpu") + model_manager.load_models([ + "models/Wan-AI/Wan2.1-T2V-1.3B/diffusion_pytorch_model.safetensors", + "models/Wan-AI/Wan2.1-T2V-1.3B/models_t5_umt5-xxl-enc-bf16.pth", + "models/Wan-AI/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth", + ]) + pipe = WanVideoRelitlivePipeline.from_model_manager(model_manager, device="cuda") + + # Initialize additional modules introduced in relit live + dim=pipe.dit.blocks[0].self_attn.q.weight.shape[0] + for block in pipe.dit.blocks: + block.env_encoder = nn.Sequential( + nn.Conv3d( + 48, dim, kernel_size=(1,2,2), stride=(1,2,2)), + nn.SiLU() + ) + block.env_encoder[0].weight.data.zero_() + block.env_encoder[0].bias.data.zero_() + block.projector = nn.Linear(dim, dim) + block.projector.weight = nn.Parameter(torch.eye(dim)) + block.projector.bias = nn.Parameter(torch.zeros(dim)) + + # if args.use_ref_image: + dim = pipe.dit.patch_embedding.out_channels + pipe.dit.ref_conv = nn.Conv2d(16, dim, kernel_size=(2,2), stride=(2,2)) + + # Load checkpoint + state_dict = torch.load(args.ckpt_path, map_location="cpu") + pipe.dit.load_state_dict(state_dict, strict=False) # True + pipe.to("cuda") + pipe.to(dtype=torch.bfloat16) + + output_dir = os.path.join(args.output_dir) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + dataset = PBRVideo_img_Dataset( + base_path=args.dataset_path, + max_num_frames=args.num_frames, + frame_interval=args.frame_interval, + num_frames=args.num_frames, + height=args.height, + width=args.width, + env_map_path=args.env_map_path, + dataset_type=args.dataset_type, + use_ref_image=args.use_ref_image, + full_resolution=args.full_resolution, + padding_resolution=args.padding_resolution, + drop_mr=args.drop_mr, + args=args, + ) + + dataloader = torch.utils.data.DataLoader( + dataset, + shuffle=False, + batch_size=1, + num_workers=args.dataloader_num_workers + ) + + # Inference + ldr_img, log_img, ref_img = None, None, None + for batch_idx, batch in enumerate(dataloader): + + input_rgb = batch["source_video"] # b c t h w + basecolor = batch["basecolor"] + depth = batch["depth"] + metallic = batch["metallic"] + normal = batch["normal"] + roughness = batch["roughness"] + ldr = batch["ldr"] + hdr_log = batch["hdr_log"] + env_dir = batch["env_dir"] + dir_path = batch["path"][0] + prompt = batch["prompt"] + print(f'processing {dir_path}, prompt: {prompt}.') + + if args.h_rotate_light != 0: + for frame_idx in range(args.num_frames): + ldr_now = np.array(ldr[0,:,frame_idx,...].permute(1, 2, 0)) # c h w -> h w c + hdr_log_now = np.array(hdr_log[0,:,frame_idx,...].permute(1, 2, 0)) + ldr_now = rotate_panorama_around_horizontal_axis(ldr_now, args.h_rotate_light) + hdr_log_now = rotate_panorama_around_horizontal_axis(hdr_log_now, args.h_rotate_light) + ldr[:,:,frame_idx:frame_idx+1,...] = torch.tensor(ldr_now).permute(2, 0, 1).unsqueeze(0).unsqueeze(2) + hdr_log[:,:,frame_idx:frame_idx+1,...] = torch.tensor(hdr_log_now).permute(2, 0, 1).unsqueeze(0).unsqueeze(2) + batch["ldr"] = ldr + batch["hdr_log"] = hdr_log + elif args.w_rotate_light != 0: + for frame_idx in range(args.num_frames): + ldr_now = np.array(ldr[0,:,frame_idx,...].permute(1, 2, 0)) # c h w -> h w c + hdr_log_now = np.array(hdr_log[0,:,frame_idx,...].permute(1, 2, 0)) + ldr_rotate = np.concatenate((ldr_now[:,args.w_rotate_light:], ldr_now[:,:args.w_rotate_light]), axis=1) + hdr_log_rotate = np.concatenate((hdr_log_now[:,args.w_rotate_light:], hdr_log_now[:,:args.w_rotate_light]), axis=1) + ldr[:,:,frame_idx:frame_idx+1,...] = torch.tensor(ldr_rotate).permute(2, 0, 1).unsqueeze(0).unsqueeze(2) + hdr_log[:,:,frame_idx:frame_idx+1,...] = torch.tensor(hdr_log_rotate).permute(2, 0, 1).unsqueeze(0).unsqueeze(2) + batch["ldr"] = ldr + batch["hdr_log"] = hdr_log + + if (args.use_fixed_frame_and_w_rotate_light or args.use_fixed_frame_and_h_rotate_light or args.use_rotate_light) and args.num_frames == 1: + continue + + if (args.use_fixed_frame_and_w_rotate_light or args.use_fixed_frame_and_h_rotate_light or args.use_rotate_light) and args.num_frames != 1: + # Repeat the first frame for num_frames times + if not args.use_rotate_light: + input_rgb = input_rgb[:, :, :1, :, :].repeat(1, 1, args.num_frames, 1, 1) + batch["source_video"] = input_rgb + basecolor = basecolor[:, :, :1, :, :].repeat(1, 1, args.num_frames, 1, 1) + batch["basecolor"] = basecolor + depth = depth[:, :, :1, :, :].repeat(1, 1, args.num_frames, 1, 1) + batch["depth"] = depth + metallic = metallic[:, :, :1, :, :].repeat(1, 1, args.num_frames, 1, 1) + batch["metallic"] = metallic + normal = normal[:, :, :1, :, :].repeat(1, 1, args.num_frames, 1, 1) + batch["normal"] = normal + roughness = roughness[:, :, :1, :, :].repeat(1, 1, args.num_frames, 1, 1) + batch["roughness"] = roughness + + if args.use_fixed_frame_and_h_rotate_light: + ldr_first = np.array(ldr[0,:,0,...].permute(1, 2, 0)) # c h w -> h w c + hdr_log_first = np.array(hdr_log[0,:,0,...].permute(1, 2, 0)) + x_rotate_list = [int(360 / (args.num_frames-1) * (i+1)) for i in range(args.num_frames-1)] + for frame_idx, x_rotate in enumerate(x_rotate_list): + ldr_now = rotate_panorama_around_horizontal_axis(ldr_first, x_rotate) + hdr_log_now = rotate_panorama_around_horizontal_axis(hdr_log_first, x_rotate) + ldr[:,:,frame_idx+1:frame_idx+2,...] = torch.tensor(ldr_now).permute(2, 0, 1).unsqueeze(0).unsqueeze(2) + hdr_log[:,:,frame_idx+1:frame_idx+2,...] = torch.tensor(hdr_log_now).permute(2, 0, 1).unsqueeze(0).unsqueeze(2) + batch["ldr"] = ldr + batch["hdr_log"] = hdr_log + else: + ldr_first = ldr[:,:,:1,...] + hdr_log_first = hdr_log[:,:,:1,...] + _, c, _, h, w = ldr_first.shape + y_rotate_list = [int(w / (args.num_frames-1) * (i+1)) for i in range(args.num_frames-1)] + for frame_idx, y_rotate in enumerate(y_rotate_list): + try: + ldr_now = torch.zeros_like(ldr_first) + ldr_now[:, :, :, :, -y_rotate:] = ldr_first[:, :, :, :, :y_rotate] + ldr_now[:, :, :, :, :-y_rotate] = ldr_first[:, :, :, :, y_rotate:] + hdr_log_now = torch.zeros_like(hdr_log_first) + hdr_log_now[:, :, :, :, -y_rotate:] = hdr_log_first[:, :, :, :, :y_rotate] + hdr_log_now[:, :, :, :, :-y_rotate] = hdr_log_first[:, :, :, :, y_rotate:] + ldr[:,:,frame_idx+1:frame_idx+2,...] = ldr_now + hdr_log[:,:,frame_idx+1:frame_idx+2,...] = hdr_log_now + except: + pdb.set_trace() + batch["ldr"] = ldr + batch["hdr_log"] = hdr_log + + seq_name = os.path.basename(dir_path) + + if args.use_rotate_light: + seq_name += "_w_rotate_light" + elif args.use_fixed_frame_and_w_rotate_light: + seq_name += "_fixed_frame_and_w_rotate_light" + elif args.use_fixed_frame_and_h_rotate_light: + seq_name += "_fixed_frame_and_h_rotate_light" + + model_name = os.path.basename(args.ckpt_path).replace("ckpt", f'{seq_name}_{args.height}_{args.width}') + + if not args.use_ref_image: + model_name += f'_no_ref_image' + elif args.use_muti_ref_image: + model_name += f'_ref_muti_image' + else: + model_name += f'_ref_mid_image' + + if args.env_map_path is not None: + env_str = os.path.basename(args.env_map_path) + model_name += f'_envdir_{env_str}' + + if args.drop_mr: + model_name += f'_drop_mr' + + if args.padding_resolution: + model_name += f'_padding' + + if args.full_resolution: + model_name += f'_full' + + if args.num_inference_steps != 50: + model_name += f'_steps{args.num_inference_steps}' + + if args.wo_ref_weight != 0.0: + args.wo_ref_weight = round(args.wo_ref_weight, 2) + model_name += f'_worw{args.wo_ref_weight}' + + if args.h_rotate_light != 0: + model_name += f'_h-rotation-{args.h_rotate_light}' + + if args.w_rotate_light != 0: + model_name += f'_w-rotation-{args.w_rotate_light}' + + if args.ref_image_path_with_idddx is not None: + model_name += f'_ref_hr_{batch_idx}' + + model_name += f'_frames{args.num_frames}' + + if args.num_frames in [1]: + save_path = os.path.join(output_dir, f"{model_name}_cfg{args.cfg_scale}_render.png") + else: + save_path = os.path.join(output_dir, f"{model_name}_cfg{args.cfg_scale}_video.mp4") + + video, envs = pipe( + prompt=prompt, + negative_prompt="", + batch=batch, + height=args.height, + width=args.width, + num_frames=args.num_frames, + cfg_scale=args.cfg_scale, + num_inference_steps=args.num_inference_steps, + seed=0, tiled=True, wo_ref_weight=args.wo_ref_weight, use_muti_ref_image=args.use_muti_ref_image + ) + + input_rgb = pipe.tensor2video(input_rgb[0]) + basecolor = pipe.tensor2video(basecolor[0]) + depth = pipe.tensor2video(depth[0]) + metallic = pipe.tensor2video(metallic[0]) + normal = pipe.tensor2video(normal[0]) + roughness = pipe.tensor2video(roughness[0]) + ldr = pipe.tensor2video(ldr[0]) + hdr_log = pipe.tensor2video(hdr_log[0]) + env_dir = pipe.tensor2video(env_dir[0]) + + if args.use_ref_image: + ref_image = batch["ref_image"].repeat(1, 1, args.num_frames, 1, 1) # b c t h w + ref_image = pipe.tensor2video(ref_image[0]) + stitched_results = stitch_frames([basecolor, metallic, roughness, depth, normal, ldr, ref_image, video, input_rgb, env_dir, envs, env_dir], rows=4, cols=3) + else: + stitched_results = stitch_frames([basecolor, metallic, roughness, depth, normal, ldr, env_dir, video, input_rgb, env_dir, envs, env_dir], rows=4, cols=3) + + print(f'Finish the inference of {model_name}.') + + if args.num_frames in [1]: + output = video[0] + output_all = stitched_results[0] + output.save(os.path.join(output_dir, f"{model_name}_cfg{args.cfg_scale}_render.png")) + output_all.save(os.path.join(output_dir, f"{model_name}_cfg{args.cfg_scale}.png")) + else: + save_video(video, os.path.join(output_dir, f"{model_name}_cfg{args.cfg_scale}_video.mp4"), fps=30, quality=args.quality) + save_video(stitched_results, os.path.join(output_dir, f"{model_name}_cfg{args.cfg_scale}.mp4"), fps=30, quality=5) + + if args.output_path is not None: + if '.mp4' in args.output_path: + save_video(video, args.output_path, fps=30, quality=args.quality) + elif '.png' in args.output_path: + output = video[0] + output.save(args.output_path) + else: + print(f'Error output_path: {args.output_path}') \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a59a5b1bb37c08a66c8aa58320b25f41632c1c8c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +torch>=2.0.0 +torchvision +cupy-cuda12x +transformers +controlnet-aux==0.0.7 +imageio +imageio[ffmpeg] +safetensors +einops +sentencepiece +protobuf +modelscope +ftfy +pynvml +accelerate +lightning +pandas +websockets +pyexr +natsort +dashscope \ No newline at end of file diff --git a/run_full_inference_gradio.py b/run_full_inference_gradio.py new file mode 100644 index 0000000000000000000000000000000000000000..5844c1d82d821806bd0d4fe9842564dc9a1a61e3 --- /dev/null +++ b/run_full_inference_gradio.py @@ -0,0 +1,1221 @@ +import gradio as gr +import os +import subprocess +import uuid +import shutil +import signal +from pathlib import Path +from PIL import Image +from datetime import datetime +import socket + +# -------------------------- Core Configuration -------------------------- +BASE_UPLOAD_DIR = "./datasets/gradio_data/upload_data" +BASE_RESULT_DIR = "./datasets/gradio_data/results" +# Demo Images Directory Configuration +DEMO_IMAGES_DIR = "./datasets/gradio_data/assets/images_demo" +BASH_SCRIPT_PATH = "./tools/full_inference_modules_gradio.sh" + +# Supported Image and Video Formats +SUPPORTED_IMAGE_FORMATS = [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".hdr"] +SUPPORTED_VIDEO_FORMATS = [".mp4", ".mov", ".avi", ".mkv"] +SUPPORTED_HDR_FORMATS = [".hdr", ".exr", ".jpg", ".png"] + +# Built-in Environment Options +ASSETS_ENV_DIR = "./datasets/gradio_data/assets/envs_demo" +BUILTIN_ENV_OPTIONS = [] +if os.path.exists(ASSETS_ENV_DIR): + for ext in SUPPORTED_HDR_FORMATS: + BUILTIN_ENV_OPTIONS.extend([f.stem for f in Path(ASSETS_ENV_DIR).glob(f"*{ext}")]) + +# Module 1 Result Types (Fixed Directory Structure) +MODULE1_RESULT_TYPES = { + "base_color": { + "name": "Base Color", + "dir": "Base Color", # Corresponding directory name + "glob_pattern": "frame_*" # File name matching pattern + }, + "normal": { + "name": "Normal Map", + "dir": "normal", + "glob_pattern": "frame_*" + }, + "roughness": { + "name": "Roughness Map", + "dir": "Roughness", + "glob_pattern": "frame_*" + } +} + +# -------------------------- Visualization Window Configuration (Adaptive Width) -------------------------- +# Base Height Configuration (Ensure Vertical Proportion) +INPUT_PREVIEW_HEIGHT = 240 # Input preview height +MODULE1_VIS_HEIGHT = 180 # Module 1 visualization height (3 results split equally horizontally) +MODULE2_VIS_HEIGHT = 200 # Module 2 visualization height (2 videos split equally horizontally) +MODULE3_VIS_HEIGHT = 400 # Module 3 visualization height (1 result full width) - Adjusted to 400px +DEMO_IMAGE_HEIGHT = 200 # Demo image display height (Increased for carousel mode) + +# -------------------------- Global Process State (For Stop Functionality) -------------------------- +# Store current running process IDs for each module (module_num: pid) +process_state = gr.State(value={}) +# Flag to indicate if one-click run should be stopped +one_click_stop_flag = gr.State(value=False) + +# -------------------------- Utility Functions (Preserve Original Logic) -------------------------- +def get_server_ip(): + """Get server's public IP for easy access""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + local_ip = s.getsockname()[0] + s.close() + return local_ip + except: + return "Unknown IP (please check server public IP manually)" + +def generate_test_id(flag): + """Generate TEST_ID with timestamp, format: flag_YYYYMMDDHHMMSS_random8chars""" + # Get current timestamp (accurate to seconds, format: YYYYMMDDHHMMSS) + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + # Keep the original 8-character random string to ensure uniqueness + random_str = uuid.uuid4().hex[:8] + # Combine to form the new TEST_ID + return f"{flag}_{timestamp}_{random_str}" + +def init_test_dir(test_id): + upload_dir = os.path.join(BASE_UPLOAD_DIR, test_id) + result_dir = os.path.join(BASE_RESULT_DIR, test_id) + os.makedirs(upload_dir, exist_ok=True) + os.makedirs(result_dir, exist_ok=True) + +def clear_test_dir(test_id): + upload_dir = os.path.join(BASE_UPLOAD_DIR, test_id) + result_dir = os.path.join(BASE_RESULT_DIR, test_id) + if os.path.exists(upload_dir): + shutil.rmtree(upload_dir) + if os.path.exists(result_dir): + shutil.rmtree(result_dir) + os.makedirs(upload_dir, exist_ok=True) + +def convert_to_jpg(file_path, save_path): + try: + with Image.open(file_path) as img: + if img.mode in ("RGBA", "P"): + img = img.convert("RGB") + img.save(save_path, "JPEG", quality=95) + return True + except Exception as e: + print(f"Image format conversion failed: {e}") + return False + +def save_uploaded_file(file, test_id, is_env=False): + if file is None: + return None, None, None + + upload_dir = os.path.join(BASE_UPLOAD_DIR, test_id) + os.makedirs(upload_dir, exist_ok=True) + + file_suffix = Path(file.name).suffix.lower() if hasattr(file, 'name') else Path(file).suffix.lower() + original_path = file.name if hasattr(file, 'name') else file + + if is_env: + env_filename = f"env_{uuid.uuid4().hex[:4]}" + save_path = os.path.join(upload_dir, f"{env_filename}{file_suffix}") + shutil.copy(original_path, save_path) + final_suffix = file_suffix + test_env = env_filename + else: + base_filename = test_id + if file_suffix in [".jpg", ".jpeg", ".png"]: + save_path = os.path.join(upload_dir, f"{base_filename}.jpg") + if not convert_to_jpg(original_path, save_path): + shutil.copy(original_path, save_path) + final_suffix = ".jpg" + else: + save_path = os.path.join(upload_dir, f"{base_filename}{file_suffix}") + shutil.copy(original_path, save_path) + final_suffix = file_suffix + test_env = None + + return final_suffix, save_path, test_env + +def clear_module_results(test_id, module_num, test_env=""): + try: + if module_num == 1: + module1_paths = [ + os.path.join(BASE_RESULT_DIR, test_id, "frames"), + # os.path.join(BASE_RESULT_DIR, test_id, "frames_delighting"), + # os.path.join(BASE_RESULT_DIR, test_id, "rego") + ] + for path in module1_paths: + if os.path.exists(path): + shutil.rmtree(path) + elif module_num == 2: + module2_path = os.path.join(BASE_RESULT_DIR, test_id, "envs", test_env) + if os.path.exists(module2_path): + shutil.rmtree(module2_path) + # elif module_num == 3: + # module3_files = list(Path(BASE_RESULT_DIR, test_id).glob(f"{test_id}.{test_env}.*")) + # module3_dir = os.path.join(BASE_RESULT_DIR, test_id, f"relighting.{test_env}") + + # for file in module3_files: + # if file.exists(): + # os.remove(file) + # if os.path.exists(module3_dir): + # shutil.rmtree(module3_dir) + return True + except Exception as e: + print(f"Failed to clear Module {module_num} results: {str(e)}") + return False + +def get_result_files(test_id, params): + results = { + "input_file": os.path.join(BASE_UPLOAD_DIR, test_id, f"{test_id}{'.jpg' if params['test_type']==1 else '.mp4'}"), + "input_file_type": "image" if params["test_type"] == 1 else "video", + "module1": { + "base_color": None, + "normal": None, + "roughness": None, + "status": "Not Executed", + "exists": False, + "missing_types": [] + }, + "module2": { + "ldr_video": None, + "env_dir_video": None, + "status": "Not Executed", + "exists": False, + "main_type": "video", + "secondary_type": "video" + }, + "module3": { + "final": None, + "status": "Not Executed", + "exists": False, + "file_type": None + }, + "test_id": test_id, + "test_env": params["test_env"] + } + + # Module 1 Results + if params["infer_1"] == 1 or os.path.exists(os.path.join(BASE_RESULT_DIR, test_id, "rego")): + module1_root_dir = os.path.join(BASE_RESULT_DIR, test_id, "rego", f"{test_id}.0") + if os.path.exists(module1_root_dir): + for result_type, config in MODULE1_RESULT_TYPES.items(): + result_dir = os.path.join(module1_root_dir, config["dir"]) + if os.path.exists(result_dir): + frame_files = [] + for fmt in SUPPORTED_IMAGE_FORMATS: + pattern = f"{config['glob_pattern']}{fmt}" + frame_files.extend(list(Path(result_dir).rglob(pattern))) + if frame_files: + frame_files.sort() + results["module1"][result_type] = str(frame_files[0]) + else: + results["module1"]["missing_types"].append(config["name"]) + else: + results["module1"]["missing_types"].append(config["name"]) + + has_result = any([ + results["module1"]["base_color"], + results["module1"]["normal"], + results["module1"]["roughness"] + ]) + results["module1"]["exists"] = has_result + + if not results["module1"]["missing_types"]: + results["module1"]["status"] = "Execution Successful (All 3 results generated)" + elif has_result: + missing_str = ", ".join(results["module1"]["missing_types"]) + results["module1"]["status"] = f"Execution Successful (Partial results missing: {missing_str})" + else: + results["module1"]["status"] = "Execution Failed: No results generated" + else: + results["module1"]["status"] = "Execution Failed: Core directory not found" + + # Module 2 Results + if params["infer_2"] == 1 or os.path.exists(os.path.join(BASE_RESULT_DIR, test_id, "envs", params["test_env"])): + env_dir = os.path.join(BASE_RESULT_DIR, test_id, "envs", params["test_env"]) + if os.path.exists(env_dir): + ldr_video = None + for fmt in SUPPORTED_VIDEO_FORMATS: + video_path = os.path.join(env_dir, f"ldr_video_fix_first_frame{fmt}") + if os.path.exists(video_path): + ldr_video = video_path + break + + env_dir_video = None + for fmt in SUPPORTED_VIDEO_FORMATS: + video_path = os.path.join(env_dir, f"env_dir_video_fix_first_frame{fmt}") + if os.path.exists(video_path): + env_dir_video = video_path + break + + results["module2"]["ldr_video"] = ldr_video + results["module2"]["env_dir_video"] = env_dir_video + results["module2"]["exists"] = True if (ldr_video or env_dir_video) else False + + if ldr_video: + results["module2"]["status"] = "Execution Successful (LDR video generated)" + elif env_dir_video: + results["module2"]["status"] = "Execution Successful (Only environment direction video generated, no LDR video)" + else: + results["module2"]["status"] = "Execution Failed: No video files generated" + else: + results["module2"]["status"] = "Execution Failed: Directory not found" + + # Module 3 Results + if params["infer_3"] == 1 or any( + os.path.exists(os.path.join(BASE_RESULT_DIR, test_id, f"{test_id}.{params['test_env']}{fmt}")) + for fmt in SUPPORTED_IMAGE_FORMATS + SUPPORTED_VIDEO_FORMATS + ): + final_files = list(Path(BASE_RESULT_DIR, test_id).glob(f"{test_id}.{params['test_env']}.*")) + if final_files: + final_file = str(final_files[0]) + file_suffix = Path(final_file).suffix.lower() + results["module3"]["final"] = final_file + results["module3"]["exists"] = True + if file_suffix in SUPPORTED_IMAGE_FORMATS: + results["module3"]["file_type"] = "image" + elif file_suffix in SUPPORTED_VIDEO_FORMATS: + results["module3"]["file_type"] = "video" + results["module3"]["status"] = "Execution Successful" + else: + results["module3"]["status"] = "Execution Failed: No image/video results generated" + + return results + +def run_bash_script(params, module_num, process_state): + """Run bash script in background and store process ID (Compatible with Python <3.7)""" + test_id = params["test_id"] + env = os.environ.copy() + env.update({ + "TEST_ID": test_id, + "TEST_TYPE": str(params["test_type"]), + "TEST_ENV": params["test_env"], + "USE_OFFICE_ENV": str(params["use_office_env"]), + "FRAME": str(params["frame"]), + "FRAME_RATE": str(params["frame_rate"]), + "INFER_1": str(params["infer_1"]), + "INFER_2": str(params["infer_2"]), + "INFER_3": str(params["infer_3"]), + "ENV_STRENGTH": str(params["env_strength"]), + "NUM_INFER_STEPS": str(params["num_infer_steps"]), + "WORW": str(params["worw"]), + "LIGHT_TYPE": str(params["light_type"]), + "CUDA_VISIBLE_DEVICES": "0", + "REPO_PATH": os.getcwd() + }) + + try: + # Replace capture_output=True with stdout/stderr pipe redirection (Compatible with all Python 3 versions) + process = subprocess.Popen( + ["bash", BASH_SCRIPT_PATH], + env=env, + stdout=subprocess.PIPE, # Capture standard output + stderr=subprocess.PIPE, # Capture standard error + text=True, + preexec_fn=os.setsid # Create new process group for easy termination + ) + + # Update process state: store PID for current module + process_state[module_num] = process.pid + print(f"Module {module_num} started with PID: {process.pid}") + + # Wait for process completion and get output + stdout, stderr = process.communicate() + + # Clear PID from state after completion + if module_num in process_state: + del process_state[module_num] + + if process.returncode == 0: + print(f"Module {module_num} execution succeeded (PID: {process.pid})") + print(f"Script stdout: {stdout}") + return True, test_id, "", stdout + else: + error_msg = f"Module {module_num} execution failed (PID: {process.pid}): {stderr}\n{stdout}" + print(error_msg) + return False, test_id, error_msg, "" + except Exception as e: + error_msg = f"Module {module_num} execution error: {str(e)}" + print(error_msg) + # Clear PID if error occurs + if module_num in process_state: + del process_state[module_num] + return False, test_id, error_msg, "" + +def stop_module_execution(module_num, process_state, current_status): + """Stop running module process (Remove temporary result cleanup logic)""" + # Check if module is running + if module_num not in process_state: + return process_state, current_status + "\n⚠️ No running process found for this module" + + pid = process_state[module_num] + try: + # Terminate entire process group (including child processes) + os.killpg(os.getpgid(pid), signal.SIGTERM) + print(f"Module {module_num} process terminated (PID: {pid})") + + # Clear PID from state + del process_state[module_num] + + # Remove temporary result cleanup logic when stopping, preserve already generated files + return process_state, current_status + f"\nβœ… Module {module_num} execution stopped successfully (PID: {pid})\nℹ️ Temporary results preserved, no cleanup performed" + except ProcessLookupError: + # Process already completed + del process_state[module_num] + return process_state, current_status + "\n⚠️ Process already completed" + except Exception as e: + return process_state, current_status + f"\n❌ Failed to stop module: {str(e)}" + +# -------------------------- Carousel Related Functions EN: -------------------------- +def get_demo_images(): + """Get all image files in the demo images directory""" + demo_images = [] + if os.path.exists(DEMO_IMAGES_DIR): + for ext in SUPPORTED_IMAGE_FORMATS: + demo_images.extend(list(Path(DEMO_IMAGES_DIR).glob(f"*{ext}"))) + # Convert to string paths and sort + demo_image_paths = [str(path) for path in sorted(demo_images)] + return demo_image_paths + +def update_carousel(index, direction, total_images): + """Update carousel index""" + if direction == "next": + new_index = (index + 1) % total_images + elif direction == "prev": + new_index = (index - 1) % total_images + else: + new_index = index + return new_index + +def get_current_image_path(index, image_paths): + """Get the image path corresponding to the current index""" + if not image_paths or index < 0 or index >= len(image_paths): + return None + return image_paths[index] + +# -------------------------- Core Business Functions (Simplified Status Output) -------------------------- +def init_input_file(input_file): + if input_file is None: + return gr.update(value=None), gr.update(value=None), "Error: Please upload an input image/video file!", gr.update(value={}) + + debug = False + flag = "demo" # choice from "test", "debug" and "demo" + # Generate unique TEST_ID and clear directory (Only reset during input file initialization) + if debug: + test_id = 'test_9c7f265f' + else: + test_id = generate_test_id(flag) + clear_test_dir(test_id) + + input_suffix, input_path, _ = save_uploaded_file(input_file, test_id, is_env=False) + test_type = 1 if input_suffix in [".jpg", ".jpeg", ".png"] else 0 + input_file_type = "image" if test_type == 1 else "video" + + base_params = { + "test_id": test_id, + "test_type": test_type, + "test_env": "", + "use_office_env": 1, + "frame": 25 if test_type == 0 else 1, + "frame_rate": 24, + "infer_1": 0, + "infer_2": 0, + "infer_3": 0, + "env_strength": 3.0, + "num_infer_steps": 20, + "worw": 0.0, + "light_type": 0 + } + + image_preview = input_path if input_file_type == "image" else None + video_preview = input_path if input_file_type == "video" else None + + # Simplified status output: Only core status information + return ( + image_preview, + video_preview, + f"βœ… Input file initialized successfully!\nFile type: {input_file_type}", + base_params + ) + +# -------------------------- Select Demo Image Initialization -------------------------- +def select_demo_image(image_path): + """Select a demo image and initialize""" + if not image_path or not os.path.exists(image_path): + return ( + gr.update(value=None, visible=False), + gr.update(value=None, visible=False), + "Error: Demo image not found!", + gr.update(value={}) + ) + + # Simulate file object for the original initialization function + class MockFile: + def __init__(self, path): + self.name = path + + mock_file = MockFile(image_path) + image_preview, video_preview, status, params = init_input_file(mock_file) + + image_visible = image_preview is not None + video_visible = video_preview is not None + + return ( + gr.update(value=image_preview, visible=image_visible, height=INPUT_PREVIEW_HEIGHT), + gr.update(value=video_preview, visible=video_visible, height=INPUT_PREVIEW_HEIGHT), + f"βœ… Selected demo image: {os.path.basename(image_path)}\n{status}", + params + ) + +def update_env_config(base_params, use_builtin_env, builtin_env_choice, env_file): + if not base_params: + return base_params, "Error: Please initialize input file first!" + + test_id = base_params["test_id"] + test_env = "" + use_office_env = 1 + + if not use_builtin_env and env_file is not None: + _, env_path, test_env = save_uploaded_file(env_file, test_id, is_env=True) + use_office_env = 0 + env_type = "Custom Environment" + else: + test_env = builtin_env_choice + use_office_env = 1 + env_type = f"Built-in Environment ({builtin_env_choice})" + + base_params.update({ + "test_env": test_env, + "use_office_env": use_office_env + }) + + # Simplified status output: Only configuration type + return base_params, f"βœ… Environment configuration updated successfully!\nType: {env_type}" + +def update_advanced_params(base_params, frame, frame_rate, env_strength, num_infer_steps, worw, light_type): + if not base_params: + return base_params, "Error: Please initialize input file first!" + + print("Current light_type:", light_type, "Type:", type(light_type)) + if light_type not in [0, 1, 2]: + light_type = 0 + print(f"Invalid light type, automatically set to default value 0") + + base_params.update({ + "frame": frame if base_params["test_type"] == 0 else 1, + "frame_rate": frame_rate, + "env_strength": env_strength, + "num_infer_steps": num_infer_steps, + "worw": worw, + "light_type": light_type + }) + + light_type_desc = { + 0: "Original Scene + Static Light", + 1: "Original Scene + Dynamic Light", + 2: "Fixed First Frame + Dynamic Light" + } + # Simplified status output: Only key parameter summary + return base_params, f"βœ… Advanced parameters updated successfully!\nFrame rate: {frame_rate} | Light Type: {light_type_desc[light_type]} | Inference Steps: {num_infer_steps} | Env Strength: {env_strength}" + +# -------------------------- Module Execution Functions (With Stop Support) -------------------------- +def run_single_module(module_num, params, process_state, re_run=True): + if not params: + if module_num == 1: + return None, None, None, process_state, "Error: Please initialize input file first!" + elif module_num == 2: + return None, None, process_state, "Error: Please initialize input file first!" + elif module_num == 3: + return gr.update(value=None), gr.update(value=None), process_state, "Error: Please initialize input file first!" + else: + return None, None, None, process_state, "Error: Invalid module number!" + + # Check if module is already running + if module_num in process_state: + pid = process_state[module_num] + if module_num == 1: + return None, None, None, process_state, f"⚠️ Module {module_num} is already running (PID: {pid})! Please stop it first." + elif module_num == 2: + return None, None, process_state, f"⚠️ Module {module_num} is already running (PID: {pid})! Please stop it first." + elif module_num == 3: + return gr.update(value=None), gr.update(value=None), process_state, f"⚠️ Module {module_num} is already running (PID: {pid})! Please stop it first." + + if not re_run: + results = get_result_files(params["test_id"], params) + + if module_num == 1: + status = results["module1"]["status"] + return ( + results["module1"]["base_color"] if results["module1"]["exists"] else None, + results["module1"]["normal"] if results["module1"]["exists"] else None, + results["module1"]["roughness"] if results["module1"]["exists"] else None, + process_state, + f"πŸ“‹ Show existing results: {status}" + ) + + elif module_num == 2: + status = results["module2"]["status"] + return ( + results["module2"]["ldr_video"] if results["module2"]["exists"] else None, + results["module2"]["env_dir_video"] if results["module2"]["exists"] else None, + process_state, + f"πŸ“‹ Show existing results: {status}" + ) + + elif module_num == 3: + status = results["module3"]["status"] + image_result = results["module3"]["final"] if (results["module3"]["exists"] and results["module3"]["file_type"] == "image") else None + video_result = results["module3"]["final"] if (results["module3"]["exists"] and results["module3"]["file_type"] == "video") else None + return ( + gr.update(value=image_result), + gr.update(value=video_result), + process_state, + f"πŸ“‹ Show existing results: {status}" + ) + + else: + return None, None, None, process_state, "Error: Invalid module number!" + + # Keep the logic of clearing old results when re-running (Avoid confusion between new and old results) + clear_success = clear_module_results(params["test_id"], module_num, params["test_env"]) + if not clear_success: + status_msg = f"⚠️ Failed to clear old results for Module {module_num}, still attempting execution..." + else: + status_msg = f"βœ… Cleared old results for Module {module_num}, starting re-execution..." + + if params["test_env"] == "" and (module_num == 2 or module_num == 3): + if module_num == 2: + return None, None, process_state, "Error: Please configure environment first!" + elif module_num == 3: + return gr.update(value=None), gr.update(value=None), process_state, "Error: Please configure environment first!" + + params["infer_1"] = 1 if module_num == 1 else 0 + params["infer_2"] = 1 if module_num == 2 else 0 + params["infer_3"] = 1 if module_num == 3 else 0 + + # Run script and get result (process_state is updated internally) + success, test_id, error_msg, stdout = run_bash_script(params, module_num, process_state) + results = get_result_files(test_id, params) + + if module_num == 1: + final_status = f"{status_msg}\n{results['module1']['status']}" + if error_msg: + final_status += f"\n{error_msg}" + return ( + results["module1"]["base_color"], + results["module1"]["normal"], + results["module1"]["roughness"], + process_state, + final_status + ) + + elif module_num == 2: + final_status = f"{status_msg}\n{results['module2']['status']}" + if error_msg: + final_status += f"\n{error_msg}" + return ( + results["module2"]["ldr_video"], + results["module2"]["env_dir_video"], + process_state, + final_status + ) + + elif module_num == 3: + final_status = f"{status_msg}\n{results['module3']['status']}" + if error_msg: + final_status += f"\n{error_msg}" + image_result = results["module3"]["final"] if (results["module3"]["exists"] and results["module3"]["file_type"] == "image") else None + video_result = results["module3"]["final"] if (results["module3"]["exists"] and results["module3"]["file_type"] == "video") else None + return ( + gr.update(value=image_result), + gr.update(value=video_result), + process_state, + final_status + ) + + else: + return None, None, None, process_state, "Invalid module number" + +# -------------------------- One-Click Run All Modules -------------------------- +def one_click_run_all(params, process_state, stop_flag): + """One-click run all modules: Module1 β†’ Module2 β†’ Module3 in sequence""" + # Reset stop flag + stop_flag = False + + # Pre-check + if not params: + return ( + None, None, None, + None, None, + gr.update(), gr.update(), + process_state, + "❌ One-click run failed: Input file not initialized", + "❌ One-click run failed: Input file not initialized", + "❌ One-click run failed: Input file not initialized", + stop_flag + ) + + if params["test_env"] == "": + return ( + None, None, None, + None, None, + gr.update(), gr.update(), + process_state, + "❌ One-click run failed: Environment not configured", + "❌ One-click run failed: Environment not configured", + "❌ One-click run failed: Environment not configured", + stop_flag + ) + + # Run Module 1 + m1_base, m1_normal, m1_rough, process_state, m1_status = run_single_module(1, params, process_state, re_run=True) + if stop_flag or "Failed" in m1_status or "Error" in m1_status: + return ( + m1_base, m1_normal, m1_rough, + None, None, + gr.update(), gr.update(), + process_state, + m1_status + "\n❌ Module1 execution failed, one-click run terminated", + "⏹️ Module2 not executed (previous module failed)", + "⏹️ Module3 not executed (previous module failed)", + stop_flag + ) + + # Run Module 2 + m2_ldr, m2_env, process_state, m2_status = run_single_module(2, params, process_state, re_run=True) + if stop_flag or "Failed" in m2_status or "Error" in m2_status: + return ( + m1_base, m1_normal, m1_rough, + m2_ldr, m2_env, + gr.update(), gr.update(), + process_state, + m1_status, + m2_status + "\n❌ Module2 execution failed, one-click run terminated", + "⏹️ Module3 not executed (previous module failed)", + stop_flag + ) + + # Run Module 3 + m3_img, m3_video, process_state, m3_status = run_single_module(3, params, process_state, re_run=True) + + # Final status + if "Failed" in m3_status or "Error" in m3_status: + m3_status += "\n⚠️ One-click run completed (Module3 execution failed)" + else: + m3_status += "\nπŸŽ‰ One-click run completed successfully!" + + return ( + m1_base, m1_normal, m1_rough, + m2_ldr, m2_env, + m3_img, m3_video, + process_state, + m1_status, + m2_status, + m3_status, + stop_flag + ) + +def stop_one_click_run(process_state, stop_flag, m1_status, m2_status, m3_status): + """Stop one-click run""" + stop_flag = True + + # Stop all running modules + for module_num in list(process_state.keys()): + process_state, _ = stop_module_execution(module_num, process_state, "") + + # Update status + m1_status += "\n⚠️ One-click run manually stopped" + m2_status += "\n⚠️ One-click run manually stopped" + m3_status += "\n⚠️ One-click run manually stopped" + + return ( + process_state, + stop_flag, + m1_status, + m2_status, + m3_status + ) + +# -------------------------- Single Page Layout Construction -------------------------- +with gr.Blocks(title="Relit-LiVE: Relighting Model Interactive Inference Tool") as demo: + gr.HTML(""" + + """) + + # Add public access information (minimal change) + server_ip = get_server_ip() + gr.Markdown(f""" + # Relit-LiVE: Relighting Model Interactive Inference Tool + > Left Panel (Input + Parameter Configuration) | Right Panel (Module 1 + Module 2 + Module 3) + """) + + # Global State Variables + base_params = gr.State(value={}) + process_state = gr.State(value={}) # Track running processes + one_click_stop_flag = gr.State(value=False) # One-click run stop flag + + # Get demo image list + demo_image_paths = get_demo_images() + total_images = len(demo_image_paths) + # Carousel state + current_index = gr.State(value=0 if total_images > 0 else -1) + + # Main Layout: Two Columns (Width Ratio 1:1.5, Right column wider for 3 modules) + with gr.Row(): + # -------------------------- Left Column: Input + Parameter Configuration -------------------------- + with gr.Column(scale=1, min_width=400): + gr.Markdown("## πŸ“₯ Input & Parameter Configuration") + + # 1. Input File Upload + input_file = gr.File( + label="Upload Input File (Supports jpg/png/mp4)", + file_types=[".jpg", ".jpeg", ".png", ".mp4"] + ) + init_input_btn = gr.Button("βœ… Initialize Input File", variant="primary") + + # -------------------------- Predefine Preview Components (Solve Undefined Variable Issue) -------------------------- + # Input Preview (Adaptive width, fixed height) + with gr.Row(): + input_image_preview = gr.Image( + label="Image Preview", + height=INPUT_PREVIEW_HEIGHT, + visible=False + ) + input_video_preview = gr.Video( + label="Video Preview", + height=INPUT_PREVIEW_HEIGHT, + visible=False + ) + + input_status = gr.Textbox(label="Initialization Status", lines=2) + + # -------------------------- Optimize: Demo Images as Carousel -------------------------- + gr.Markdown("### πŸ“Έ Demo Images") + + if demo_image_paths and total_images > 0: + # Carousel container + gr.HTML('
') + + # Main image display + carousel_image = gr.Image( + value=demo_image_paths[0] if total_images > 0 else None, + label="Demo Image", + height=DEMO_IMAGE_HEIGHT, + interactive=False + ) + + # Navigation buttons + with gr.Row(): + prev_btn = gr.Button("◀️ Previous", elem_classes=["carousel-btn", "carousel-prev"]) + next_btn = gr.Button("Next ▢️", elem_classes=["carousel-btn", "carousel-next"]) + + # Indicators + gr.HTML('
') + indicator_btns = [] + for i in range(total_images): + btn = gr.Button( + "", + elem_classes=["indicator-dot"] + (["active"] if i == 0 else []), + size="sm", + visible=False # Hide default button style, use CSS style + ) + indicator_btns.append(btn) + # Click indicator to switch image + btn.click( + fn=lambda idx=i: ( + idx, # Update current index + demo_image_paths[idx] if idx < len(demo_image_paths) else None # Update image + ), + inputs=[], + outputs=[current_index, carousel_image] + ) + gr.HTML('
') + + # Select current image button + select_current_btn = gr.Button( + "βœ… Select This Image", + variant="primary", + elem_classes=["select-image-btn"] + ) + + gr.HTML('
') + + # Carousel navigation logic + def update_carousel_ui(index): + """Update carousel UI""" + if index < 0 or index >= len(demo_image_paths): + return None + return demo_image_paths[index] + + # Previous button + prev_btn.click( + fn=lambda idx: update_carousel(idx, "prev", total_images), + inputs=[current_index], + outputs=[current_index] + ).then( + fn=update_carousel_ui, + inputs=[current_index], + outputs=[carousel_image] + ) + + # Next button + next_btn.click( + fn=lambda idx: update_carousel(idx, "next", total_images), + inputs=[current_index], + outputs=[current_index] + ).then( + fn=update_carousel_ui, + inputs=[current_index], + outputs=[carousel_image] + ) + + # Select current image + select_current_btn.click( + fn=lambda idx: select_demo_image(demo_image_paths[idx] if idx < len(demo_image_paths) else None), + inputs=[current_index], + outputs=[input_image_preview, input_video_preview, input_status, base_params] + ) + + else: + gr.Markdown("*No demo images found in the specified directory*") + + # 2. Environment Configuration + gr.Markdown("---\n## πŸ’‘ Environment Configuration") + use_builtin_env = gr.Checkbox( + label="Use Built-in Environment (Uncheck to upload custom)", + value=True + ) + with gr.Row(): + builtin_env_choice = gr.Dropdown( + label="Built-in Environment Selection", + choices=BUILTIN_ENV_OPTIONS, + value="Pink_Sunrise" + ) + env_file = gr.File( + label="Custom Environment (hdr/jpg/png)", + file_types=[".hdr", ".jpg", ".jpeg", ".png"], + visible=False + ) + update_env_btn = gr.Button("πŸ”„ Update Environment", variant="primary") + env_status = gr.Textbox(label="Environment Config Status", lines=1) + + # 3. Advanced Parameters + gr.Markdown("---\n## βš™οΈ Advanced Parameters") + frame = gr.Slider( + label="Video Frames (Video only, 1-57, 4n+1)", + minimum=1, maximum=57, step=4, value=25 + ) + frame_rate = gr.Slider( + label="Sample Rate of Video Frames (Video only, 10-24)", + minimum=10, maximum=24, step=1, value=24 + ) + env_strength = gr.Slider( + label="Environment Strength (0-5)", + minimum=0, maximum=5, step=0.1, value=3.0 + ) + num_infer_steps = gr.Slider( + label="Inference Steps (1-50)", + minimum=1, maximum=50, step=1, value=20 + ) + worw = gr.Slider( + label="Reference Image Weight (0-5, smaller = more influence. Increase it when the light fails.)", + minimum=0, maximum=5, step=0.1, value=0.0 + ) + light_type = gr.Radio( + label="Light Type (0=Original+Static, 1=Original+Dynamic, 2=Fixed First Frame+Dynamic)", + choices=[0, 1, 2], + value=0 + ) + update_advanced_btn = gr.Button("πŸ”„ Update Parameters", variant="primary") + advanced_status = gr.Textbox(label="Parameter Update Status", lines=2) + + # One-Click Run All Modules + gr.Markdown("---\n## πŸš€ One-Click Run All Modules") + with gr.Row(): + one_click_run_btn = gr.Button("▢️ Run Module1β†’Module2β†’Module3", variant="primary", size="lg") + one_click_stop_btn = gr.Button("⏹️ Stop All Running Modules", variant="stop", size="lg") + one_click_status = gr.Textbox(label="One-Click Run Status", lines=3, placeholder="Click button above to start one-click run...") + + # -------------------------- Right Column: Module 1 + Module 2 + Module 3 (Vertical Arrangement) -------------------------- + with gr.Column(scale=1.5, min_width=600): + # Module 1: Inverse Rendering (3 results split horizontally) + gr.Markdown("## πŸ”§ Module 1: Inverse Rendering") + with gr.Row(): + run_module1_btn = gr.Button("▢️ Start Execution", variant="primary") + stop_module1_btn = gr.Button("⏹️ Stop Execution", variant="stop") + show_module1_btn = gr.Button("πŸ“‹ Show Results", variant="secondary") + module1_status = gr.Textbox(label="Execution Status", lines=2) + + # Module 1 Visualization Results + gr.Markdown("### Partial Visualization Results (Base Color | Normal Map | Roughness Map)") + with gr.Row(equal_height=True): + module1_base_color = gr.Image(label="Base Color", height=MODULE1_VIS_HEIGHT, scale=1) + module1_normal = gr.Image(label="Normal Map", height=MODULE1_VIS_HEIGHT, scale=1) + module1_roughness = gr.Image(label="Roughness Map", height=MODULE1_VIS_HEIGHT, scale=1) + + # Module 2: Environment Processing (2 videos split horizontally) + gr.Markdown("---\n## πŸ”§ Module 2: Environment Processing") + with gr.Row(): + run_module2_btn = gr.Button("▢️ Start Execution", variant="primary") + stop_module2_btn = gr.Button("⏹️ Stop Execution", variant="stop") + show_module2_btn = gr.Button("πŸ“‹ Show Results", variant="secondary") + module2_status = gr.Textbox(label="Execution Status", lines=2) + + # Module 2 Visualization Results + gr.Markdown("### Visualization Results (LDR Video | Environment Direction Video)") + with gr.Row(equal_height=True): + module2_ldr_video = gr.Video(label="LDR Video (Core Result)", height=MODULE2_VIS_HEIGHT, scale=1) + module2_env_video = gr.Video(label="Environment Direction Video", height=MODULE2_VIS_HEIGHT, scale=1) + + # Module 3: Relighting (1 result full width) + gr.Markdown("---\n## πŸ”§ Module 3: Relighting") + with gr.Row(): + run_module3_btn = gr.Button("▢️ Start Execution", variant="primary") + stop_module3_btn = gr.Button("⏹️ Stop Execution", variant="stop") + show_module3_btn = gr.Button("πŸ“‹ Show Results", variant="secondary") + module3_status = gr.Textbox(label="Execution Status", lines=2) + + # Module 3 Visualization Results + gr.Markdown("### Visualization Results (Auto-adapt Image/Video)") + with gr.Row(): + module3_image_result = gr.Image( + label="Relighting Result (Image)", + height=MODULE3_VIS_HEIGHT, + visible=False, + scale=1 + ) + module3_video_result = gr.Video( + label="Relighting Result (Video)", + height=MODULE3_VIS_HEIGHT, + visible=False, + scale=1 + ) + + # -------------------------- Event Bindings -------------------------- + # Input File Initialization + def init_and_show_preview(input_file): + image_preview, video_preview, status, params = init_input_file(input_file) + image_visible = image_preview is not None + video_visible = video_preview is not None + return ( + gr.update(value=image_preview, visible=image_visible, height=INPUT_PREVIEW_HEIGHT), + gr.update(value=video_preview, visible=video_visible, height=INPUT_PREVIEW_HEIGHT), + status, + params + ) + + init_input_btn.click( + fn=init_and_show_preview, + inputs=[input_file], + outputs=[input_image_preview, input_video_preview, input_status, base_params] + ) + + # Environment Configuration Visibility + use_builtin_env.change( + fn=lambda x: gr.update(visible=not x), + inputs=[use_builtin_env], + outputs=[env_file] + ) + + update_env_btn.click( + fn=update_env_config, + inputs=[base_params, use_builtin_env, builtin_env_choice, env_file], + outputs=[base_params, env_status] + ) + + # Advanced Parameters Update + update_advanced_btn.click( + fn=update_advanced_params, + inputs=[base_params, frame, frame_rate, env_strength, num_infer_steps, worw, light_type], + outputs=[base_params, advanced_status] + ) + + # -------------------------- Module 1: Execution/Stop/Display -------------------------- + run_module1_btn.click( + fn=lambda params, process_state: run_single_module(1, params, process_state, re_run=True), + inputs=[base_params, process_state], + outputs=[module1_base_color, module1_normal, module1_roughness, process_state, module1_status] + ) + + stop_module1_btn.click( + fn=lambda ps, cs: stop_module_execution(1, ps, cs), + inputs=[process_state, module1_status], + outputs=[process_state, module1_status] + ) + + show_module1_btn.click( + fn=lambda params, process_state: run_single_module(1, params, process_state, re_run=False), + inputs=[base_params, process_state], + outputs=[module1_base_color, module1_normal, module1_roughness, process_state, module1_status] + ) + + # -------------------------- Module 2: Execution/Stop/Display -------------------------- + run_module2_btn.click( + fn=lambda params, process_state: run_single_module(2, params, process_state, re_run=True), + inputs=[base_params, process_state], + outputs=[module2_ldr_video, module2_env_video, process_state, module2_status] + ) + + stop_module2_btn.click( + fn=lambda ps, cs: stop_module_execution(2, ps, cs), + inputs=[process_state, module2_status], + outputs=[process_state, module2_status] + ) + + show_module2_btn.click( + fn=lambda params, process_state: run_single_module(2, params, process_state, re_run=False), + inputs=[base_params, process_state], + outputs=[module2_ldr_video, module2_env_video, process_state, module2_status] + ) + + # -------------------------- Module 3: Execution/Stop/Display -------------------------- + run_module3_btn.click( + fn=lambda params, process_state: run_single_module(3, params, process_state, re_run=True), + inputs=[base_params, process_state], + outputs=[module3_image_result, module3_video_result, process_state, module3_status] + ).then( + fn=lambda image_val, video_val, status: ( + gr.update(value=image_val, visible=image_val is not None, height=MODULE3_VIS_HEIGHT), + gr.update(value=video_val, visible=video_val is not None, height=MODULE3_VIS_HEIGHT), + status + ), + inputs=[module3_image_result, module3_video_result, module3_status], + outputs=[module3_image_result, module3_video_result, module3_status] + ) + + stop_module3_btn.click( + fn=lambda ps, cs: stop_module_execution(3, ps, cs), + inputs=[process_state, module3_status], + outputs=[process_state, module3_status] + ) + + show_module3_btn.click( + fn=lambda params, process_state: run_single_module(3, params, process_state, re_run=False), + inputs=[base_params, process_state], + outputs=[module3_image_result, module3_video_result, process_state, module3_status] + ).then( + fn=lambda image_val, video_val, status: ( + gr.update(value=image_val, visible=image_val is not None, height=MODULE3_VIS_HEIGHT), + gr.update(value=video_val, visible=video_val is not None, height=MODULE3_VIS_HEIGHT), + status + ), + inputs=[module3_image_result, module3_video_result, module3_status], + outputs=[module3_image_result, module3_video_result, module3_status] + ) + + # -------------------------- One-Click Run Event Bindings -------------------------- + one_click_run_btn.click( + fn=one_click_run_all, + inputs=[base_params, process_state, one_click_stop_flag], + outputs=[ + module1_base_color, module1_normal, module1_roughness, + module2_ldr_video, module2_env_video, + module3_image_result, module3_video_result, + process_state, + module1_status, module2_status, module3_status, + one_click_stop_flag + ] + ).then( + fn=lambda image_val, video_val: ( + gr.update(visible=image_val is not None, height=MODULE3_VIS_HEIGHT), + gr.update(visible=video_val is not None, height=MODULE3_VIS_HEIGHT) + ), + inputs=[module3_image_result, module3_video_result], + outputs=[module3_image_result, module3_video_result] + ) + + one_click_stop_btn.click( + fn=stop_one_click_run, + inputs=[process_state, one_click_stop_flag, module1_status, module2_status, module3_status], + outputs=[process_state, one_click_stop_flag, module1_status, module2_status, module3_status] + ) + +# -------------------------- Run Gradio (Public Access Enabled) -------------------------- +if __name__ == "__main__": + os.makedirs(BASE_UPLOAD_DIR, exist_ok=True) + os.makedirs(BASE_RESULT_DIR, exist_ok=True) + + # Print public access information + server_ip = get_server_ip() + print("="*60) + print("Relit-LiVE: Relighting Model Interactive Inference Tool") + print(f"πŸ“‘ Public Access URL: http://{server_ip}:7861") + print("πŸ”§ All original functions are preserved") + print("πŸ–ΌοΈ Demo images displayed in carousel (slider) layout") + print("⚠️ No authentication - use with caution") + print("="*60) + + demo.launch( + server_name="0.0.0.0", # Listen on all interfaces (public access) + server_port=7861, # Fixed port (can be modified if needed) + share=False, + show_error=True + ) \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..483a72990f1da9a2e339d24ef17b4a75e55cdfe1 --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +import os +from setuptools import setup, find_packages +import pkg_resources + +# Path to the requirements file +requirements_path = os.path.join(os.path.dirname(__file__), "requirements.txt") + +# Read the requirements from the requirements file +if os.path.exists(requirements_path): + with open(requirements_path, 'r') as f: + install_requires = [str(r) for r in pkg_resources.parse_requirements(f)] +else: + install_requires = [] + +setup( + name="diffsynth", + version="1.1.7", + description="Enjoy the magic of Diffusion models!", + author="Artiprocher", + packages=find_packages(), + install_requires=install_requires, + include_package_data=True, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ], + package_data={"diffsynth": ["tokenizer_configs/**/**/*.*"]}, + python_requires='>=3.6', +)