Upload folder using huggingface_hub

README.md

@@ -1,3 +1,14 @@
----
-license: mit
----
+---
+library_name: ss4d
+pipeline_tag: image-to-3d
+license: mit
+language:
+- en
+---
+# SS4D
+
+Official model weights for *SS4D: Native 4D Generative Model via Structured Spacetime Latents*
+
+Project page: https://lizb6626.github.io/SS4D/
+
+Code: https://github.com/Lizb6626/SS4D/

ckpts/slat_dec_gs_4d.json

@@ -0,0 +1,35 @@
+{
+    "name": "SLatGaussianDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "pe_mode": "rope_t",
+        "use_fp16": true,
+        "use_checkpoint": true,
+        "qk_rms_norm": false,
+        "expand_self_attn": true,
+        "representation_config": {
+            "lr": {
+                "_xyz": 1.0,
+                "_features_dc": 1.0,
+                "_opacity": 1.0,
+                "_scaling": 1.0,
+                "_rotation": 0.1
+            },
+            "perturb_offset": true,
+            "voxel_size": 1.5,
+            "num_gaussians": 32,
+            "2d_filter_kernel_size": 0.1,
+            "3d_filter_kernel_size": 0.0009,
+            "scaling_bias": 0.004,
+            "opacity_bias": 0.1,
+            "scaling_activation": "softplus"
+        }
+    }
+}

ckpts/slat_dec_gs_4d.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9033301cc6a56a0369e3009deb5a9fdd0528cc8f6a82e6a27bc87048a94da1e0
+size 341486240

ckpts/slat_enc_4d.json

@@ -0,0 +1,19 @@
+{
+    "name": "SLatEncoder",
+    "args": {
+        "resolution": 64,
+        "in_channels": 1024,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "pe_mode": "rope_t",
+        "use_fp16": true,
+        "use_checkpoint": true,
+        "qk_rms_norm": false,
+        "expand_self_attn": true
+    }
+}

ckpts/slat_enc_4d.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b800687c94726142a71f3f491b7fa00d3bc2adebd5accef563ee086d2bb6c55e
+size 343278088

ckpts/slat_flow_video_4d.json

@@ -0,0 +1,28 @@
+{
+    "name": "SLatFlow4DModel",
+    "args": {
+        "resolution": 64,
+        "in_channels": 8,
+        "out_channels": 8,
+        "model_channels": 1024,
+        "cond_channels": 1024,
+        "num_blocks": 24,
+        "num_heads": 16,
+        "mlp_ratio": 4,
+        "patch_size": 2,
+        "num_io_res_blocks": 2,
+        "num_io_res4d_blocks": 2,
+        "io_block_channels": [
+            128
+        ],
+        "io_block_channels_t": [
+            1024
+        ],
+        "pe_mode": "rope_t",
+        "qk_rms_norm": true,
+        "use_fp16": true,
+        "expand_self_attn": true,
+        "use_checkpoint": true,
+        "conv_type": "3D_1D"
+    }
+}

ckpts/slat_flow_video_4d.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af933cecb213c8a2a1a7a6dd3f2b14fcbdea101b764fef6f878acef16ce2528b
+size 2578072200

ckpts/ss_dec_conv3d_16l8_fp16.json

@@ -0,0 +1,12 @@
+
+{
+    "name": "SparseStructureDecoder",
+    "args": {
+        "out_channels": 1,
+        "latent_channels": 8,
+        "num_res_blocks": 2,
+        "num_res_blocks_middle": 2,
+        "channels": [512, 128, 32],
+        "use_fp16": true
+    }
+}

ckpts/ss_dec_conv3d_16l8_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c76d4a40519aa2d711cc263a8404105231ac26db31d946bed48b84fee79009a
+size 147591972

ckpts/ss_enc_conv3d_16l8_fp16.json

@@ -0,0 +1,12 @@
+
+{
+    "name": "SparseStructureEncoder",
+    "args": {
+        "in_channels": 1,
+        "latent_channels": 8,
+        "num_res_blocks": 2,
+        "num_res_blocks_middle": 2,
+        "channels": [32, 128, 512],
+        "use_fp16": true
+    }
+}

ckpts/ss_enc_conv3d_16l8_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:107874eeaa0feb82f51b19db5da7db534fb7e7f19e5a122b9ff1bc2e258bfc6d
+size 119068016

ckpts/ss_flow_video_4d.json

@@ -0,0 +1,20 @@
+{
+    "name": "SparseStructureFlow4DModel",
+    "args": {
+        "resolution": 16,
+        "in_channels": 8,
+        "out_channels": 8,
+        "model_channels": 1024,
+        "cond_channels": 1024,
+        "num_blocks": 24,
+        "num_heads": 16,
+        "mlp_ratio": 4,
+        "patch_size": 1,
+        "pe_mode": "rope_t",
+        "qk_rms_norm": true,
+        "use_fp16": true,
+        "expand_self_attn": true,
+        "use_checkpoint": true,
+        "window_size": 8
+    }
+}

ckpts/ss_flow_video_4d.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e65734996e541660da54c5f15c0cf674cec738edeb35d6c7f2a508869f25ca06
+size 2239002104

pipeline.json

@@ -0,0 +1,58 @@
+{
+    "name": "TrellisVideoTo4DPipeline",
+    "args": {
+        "models": {
+            "sparse_structure_decoder": "ckpts/ss_dec_conv3d_16l8_fp16",
+            "sparse_structure_flow_model": "ckpts/ss_flow_video_4d",
+            "slat_decoder_gs": "ckpts/slat_dec_gs_4d",
+            "slat_flow_model": "ckpts/slat_flow_video_4d"
+        },
+        "sparse_structure_sampler": {
+            "name": "FlowEulerGuidanceIntervalSampler",
+            "args": {
+                "sigma_min": 1e-5
+            },
+            "params": {
+                "steps": 25,
+                "cfg_strength": 5.0,
+                "cfg_interval": [0.5, 1.0],
+                "rescale_t": 3.0
+            }
+        },
+        "slat_sampler": {
+            "name": "FlowEulerGuidanceIntervalSampler",
+            "args": {
+                "sigma_min": 1e-5
+            },
+            "params": {
+                "steps": 25,
+                "cfg_strength": 5.0,
+                "cfg_interval": [0.5, 1.0],
+                "rescale_t": 3.0
+            }
+        },
+        "slat_normalization": {
+            "mean": [
+                -2.826356,
+                0.33626583,
+                0.0763019,
+                -0.117598,
+                -0.4892621,
+                1.0101169,
+                -1.0808244,
+                1.4578744
+            ],
+            "std": [
+                2.8175173,
+                2.6447911,
+                2.5076609,
+                2.3946748,
+                3.438241,
+                2.864445,
+                3.2401907,
+                3.1412923
+            ]
+        },
+        "image_cond_model": "dinov2_vitl14_reg"
+    }
+}