Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+asset/teaser.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,24 @@
+---
+license: other
+library_name: DiffusionRenderer
+arxiv: 2501.18590
+tags:
+- nvidia
+- rendering
+- diffusion
+pipeline_tag: rendering
+---
+
+# DiffusionRenderer: Neural Inverse and Forward Rendering with Video Diffusion Models
+
+![img](asset/teaser.jpg)
+
+[Ruofan Liang](https://www.cs.toronto.edu/~ruofan/)\*, [Zan Gojcic](https://zgojcic.github.io/), [Huan Ling](https://www.cs.toronto.edu/~linghuan/),  [Jacob Munkberg](https://research.nvidia.com/person/jacob-munkberg), [Jon Hasselgren](https://research.nvidia.com/person/jon-hasselgren), [Zhi-Hao Lin](https://chih-hao-lin.github.io/), [Jun Gao](https://www.cs.toronto.edu/~jungao/), [Alexander Keller](https://research.nvidia.com/person/alex-keller), [Nandita Vijaykumar](https://www.cs.toronto.edu/~nandita/), [Sanja Fidler](https://www.cs.toronto.edu/~fidler/),   [Zian Wang](https://www.cs.toronto.edu/~zianwang/)\*
+
+\* indicates equal contribution
+
+**[Paper](https://arxiv.org/abs/2501.18590) | [Project Page](https://research.nvidia.com/labs/toronto-ai/DiffusionRenderer/)**
+
+**Overview.**
+DiffusionRenderer is a general-purpose framework that achieves high-quality geometry and material estimation from real-world videos (inverse rendering), and photorealistic image/video synthesis from G-buffers and lighting (forward rendering). Both the inverse and forward renderers are video diffusion models trained on a combination of curated synthetic datasets and auto-labeled real-world videos.
+Unlike classic PBR, which relies on precise geometry and path tracing, DiffusionRenderer provides a data-driven approximation of light transport through video GenAI models. It synthesizes realistic lighting effects without explicit simulation, complementing PBR for real-world applications such as relighting and material editing, especially when explicit geometry is inaccessible or inaccurate.

model_index.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "RGBXVideoDiffusionPipeline",
+  "_diffusers_version": "0.28.0",
+  "feature_extractor": [
+    null,
+    null
+  ],
+  "image_encoder": [
+    null,
+    null
+  ],
+  "scheduler": [
+    "diffusers",
+    "EulerDiscreteScheduler"
+  ],
+  "unet": [
+    "src.models.custom_unet_st",
+    "UNetCustomSpatioTemporalConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKLTemporalDecoder"
+  ]
+}

scheduler/scheduler_config.json

@@ -0,0 +1,22 @@
+{
+  "_class_name": "EulerDiscreteScheduler",
+  "_diffusers_version": "0.28.0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "final_sigmas_type": "zero",
+  "interpolation_type": "linear",
+  "num_train_timesteps": 1000,
+  "prediction_type": "v_prediction",
+  "rescale_betas_zero_snr": false,
+  "set_alpha_to_one": false,
+  "sigma_max": 700.0,
+  "sigma_min": 0.002,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "timestep_spacing": "trailing",
+  "timestep_type": "continuous",
+  "trained_betas": null,
+  "use_karras_sigmas": true
+}

unet/config.json

@@ -0,0 +1,40 @@
+{
+  "_class_name": "UNetCustomSpatioTemporalConditionModel",
+  "_diffusers_version": "0.28.0",
+  "_name_or_path": "../logs/checkpoint-38000",
+  "addition_time_embed_dim": 256,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "context_embedding_type": "clip",
+  "context_vocab_size": 6,
+  "cross_attention_dim": 1024,
+  "down_block_types": [
+    "CrossAttnDownBlockSpatioTemporal",
+    "CrossAttnDownBlockSpatioTemporal",
+    "CrossAttnDownBlockSpatioTemporal",
+    "DownBlockSpatioTemporal"
+  ],
+  "in_channels": 8,
+  "layers_per_block": 2,
+  "num_attention_heads": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "num_frames": 14,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": 768,
+  "sample_size": 96,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "UpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal",
+    "CrossAttnUpBlockSpatioTemporal"
+  ]
+}

unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9552d85fcac9d32ef643ba9f8ed663386706a3797162c1e844e26b19983ee84
+size 6098707136

vae/config.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "AutoencoderKLTemporalDecoder",
+  "_diffusers_version": "0.28.0",
+  "_name_or_path": "/lustre/fsw/portfolios/nvr/projects/nvr_torontoai_holodeck/huggingface_home/hub/models--stabilityai--stable-video-diffusion-img2vid/snapshots/9cf024d5bfa8f56622af86c884f26a52f6676f2e/",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "out_channels": 3,
+  "sample_size": 768,
+  "scaling_factor": 0.18215
+}

vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af602cd0eb4ad6086ec94fbf1438dfb1be5ec9ac03fd0215640854e90d6463a3
+size 195531910