Upload folder using huggingface_hub

ckpts/hdri_encoder_4096tokens_fp16.json

@@ -0,0 +1,11 @@
+{
+    "name": "Hdri_Encoder",
+    "args": {
+        "output_dim": 768,
+        "num_tokens": 4096,
+        "cnn_out_channels": 256,
+        "n_heads": 12,
+        "num_transformer_layers": 3,
+        "rope_base": 10000.0
+    }
+}

ckpts/hdri_encoder_4096tokens_fp16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c763776c1fe8d4857283356349f26bcd286b8f631d4e3951f087348b031af052
+size 868756605

ckpts/neural_basis_3layer_relu_fp16.json

@@ -0,0 +1,14 @@
+{
+    "name": "NeuralBasis",
+    "args":{
+      "input_dim": 32,
+      "output_dim": 5,
+      "hidden_dim": 64,
+      "num_layers": 3,
+      "use_normal": true,
+      "activation": "relu",
+      "normal_dim": 3,
+      "num_frequencies": 4,
+      "use_residual": false
+    }
+}

ckpts/neural_basis_3layer_relu_fp16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0b4ca2ec379c93784841ed3be7c7042117accfa3d93c9e893e9fcbccc51d022
+size 840373734

ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16.json

@@ -0,0 +1,16 @@
+{
+    "name": "SLatGaussianDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "cond_channels": 768,
+        "num_blocks": 12,
+        "num_register_tokens": 16,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true
+    }
+}

ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eda01884ae8746b44d977aff0514287224bc7f63bedf67269c974a35db3723c
+size 840401603

ckpts/slat_flow_img_dit_L_64l8p2_fp16.json

@@ -0,0 +1,19 @@
+{
+    "name": "SLatFlowModel",
+    "args": {
+        "resolution": 64,
+        "in_channels": 8,
+        "out_channels": 8,
+        "model_channels": 1024,
+        "cond_channels": 1024,
+        "num_blocks": 24,
+        "num_heads": 16,
+        "mlp_ratio": 4,
+        "patch_size": 2,
+        "num_io_res_blocks": 2,
+        "io_block_channels": [128],
+        "pe_mode": "ape",
+        "qk_rms_norm": true,
+        "use_fp16": true
+    }
+}

ckpts/slat_flow_img_dit_L_64l8p2_fp16_18w.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:decc9df7523b0aefc87fd097acedfd6777e93e85d3b9eb74b56fffa8348a2015
+size 2401821834

ckpts/slat_flow_img_dit_L_64l8p2_fp16_23w.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d92835a8792e90687fed8c939d34959e94e73c30768d3343581a9f7b951c22ff
+size 2401821834

ckpts/slat_renderder_gs_swin8_B_64l8gs32_fp16.json

@@ -0,0 +1,42 @@
+{
+    "name": "SLatGaussianRenderer",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 768,
+        "cond_channels": 768,
+        "num_blocks": 6,
+        "num_register_tokens": 16,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "representation_config": {
+            "lr": {
+                "_xyz": 1.0,
+                "_base_color": 1.0,
+                "_roughness": 1.0,
+                "_metallic": 1.0,
+                "_scaling": 1.0,
+                "_scaling_view": 1.0,
+                "_rotation": 0.2,
+                "_opacity": 1.0,
+                "_shadow": 0.5,
+                "_brightness": 0.5,
+                "_pbr1": 0.5,
+                "_hdri1": 0.5,
+                "_hdri2": 0.5,
+                "_rgb": 1.0
+            },
+            "perturb_offset": true,
+            "voxel_size": 1.5,
+            "num_gaussians": 32,
+            "2d_filter_kernel_size": 0.1,
+            "3d_filter_kernel_size": 9e-4,
+            "scaling_bias": 4e-3,
+            "opacity_bias": 0.1,
+            "scaling_activation": "softplus"
+        }
+    }
+}

ckpts/slat_renderder_gs_swin8_B_64l8gs32_fp16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9c8f5fc1acb8477656bf5391f92051c80dfe212d8f0035207d92d8966573547
+size 840392413

pipeline.yaml

@@ -0,0 +1,51 @@
+name: NeARImageToRelightable3DPipeline
+args:
+  models:
+    decoder: ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16
+    hdri_encoder: ckpts/hdri_encoder_4096tokens_fp16
+    neural_basis: ckpts/neural_basis_3layer_relu_fp16
+    renderer: ckpts/slat_renderder_gs_swin8_B_64l8gs32_fp16
+    slat_flow_model: ckpts/slat_flow_img_dit_L_64l8p2_fp16
+
+  hyshape:
+    model_id: tencent/Hunyuan3D-2.1 # or use custom local path if you have downloaded the model locally https://huggingface.co/tencent/Hunyuan3D-2.1/tree/main/hunyuan3d-dit-v2-1
+
+  # ultrashape:
+  #   model_id: infinith/UltraShape # or use custom local path if you have downloaded the model locally https://huggingface.co/infinith/UltraShape
+
+  sr_model:
+    model_path: weights/RealESRGAN_x4plus.pth
+
+  slat_sampler:
+    name: FlowEulerGuidanceIntervalSampler
+    args:
+      sigma_min: 0.00001
+    params:
+      steps: 25
+      cfg_strength: 5.0
+      cfg_interval:
+        - 0.5
+        - 1.0
+      rescale_t: 3.0
+
+  slat_normalization:
+    mean:
+      - -2.1687545776367188
+      - -0.004347046371549368
+      - -0.13352349400520325
+      - -0.08418072760105133
+      - -0.5271206498146057
+      - 0.7238689064979553
+      - -1.1414450407028198
+      - 1.2039363384246826
+    std:
+      - 2.377650737762451
+      - 2.386378288269043
+      - 2.124418020248413
+      - 2.1748552322387695
+      - 2.663944721221924
+      - 2.371192216873169
+      - 2.6217446327209473
+      - 2.684523105621338
+
+  image_cond_model: dinov2_vitl14_reg

weights/RealESRGAN_x4plus.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fa0d38905f75ac06eb49a7951b426670021be3018265fd191d2125df9d682f1
+size 67040989