Upload 17 files

README.md

@@ -1,3 +1,16 @@
----
-license: mit
----
+---
+library_name: reconviagen
+pipeline_tag: mvimages-to-3d
+license: mit
+language:
+- en
+---
+# trellis vggt v0-1
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+ReconViaGen was introduced in the paper [ReconViaGen: Towards Accurate Multi-view 3D Object Reconstruction via Generation].
+
+Project page: https://jiahao620.github.io/reconviagen/
+
+Code: https://github.com/GAP-LAB-CUHK-SZ/ReconViaGen

ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16.json

@@ -0,0 +1,31 @@
+{
+    "name": "SLatGaussianDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "representation_config": {
+            "lr": {
+                "_xyz": 1.0,
+                "_features_dc": 1.0,
+                "_opacity": 1.0,
+                "_scaling": 1.0,
+                "_rotation": 0.1
+            },
+            "perturb_offset": true,
+            "voxel_size": 1.5,
+            "num_gaussians": 32,
+            "2d_filter_kernel_size": 0.1,
+            "3d_filter_kernel_size": 9e-4,
+            "scaling_bias": 4e-3,
+            "opacity_bias": 0.1,
+            "scaling_activation": "softplus"
+        }
+    }
+}

ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c84bcef5ce0af1f48b1b5558dabc7575a13346043c41a7e0610f1fa619a161
+size 171450952

ckpts/slat_dec_mesh_swin8_B_64l8m256c_fp16.json

@@ -0,0 +1,17 @@
+{
+    "name": "SLatMeshDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "representation_config": {
+            "use_color": true
+        }
+    }
+}

ckpts/slat_dec_mesh_swin8_B_64l8m256c_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e87aba94b5786407eb06d0502c1ed0885a0027a3f2b8537bfe15b0a92c01859
+size 181903412

ckpts/slat_flow_img_dit_L_64l8p2_fp16.json

@@ -0,0 +1,18 @@
+{
+    "name": "SLatFlowModel",
+    "args": {
+        "resolution": 64,
+        "in_channels": 8,
+        "out_channels": 8,
+        "model_channels": 1024,
+        "cond_channels": 1024,
+        "num_blocks": 24,
+        "num_heads": 16,
+        "mlp_ratio": 4,
+        "patch_size": 2,
+        "num_io_res_blocks": 2,
+        "io_block_channels": [128],
+        "pe_mode": "ape",
+        "qk_rms_norm": true
+    }
+}

ckpts/slat_flow_img_dit_L_64l8p2_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9907b5e50f46e55601ce72419650f8ace13cdfd8acb8b209c7b2622fe4bfd2c6
+size 1203755136

ckpts/slat_vggt_cond.json

@@ -0,0 +1,8 @@
+{
+    "name": "ModulatedSLATMultiViewCond",
+    "args": {
+        "channels": 1024,
+        "ctx_channels": 3072,
+        "use_fp16": true
+    }
+}

ckpts/slat_vggt_cond.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18945872ec1a5fc13b2bfe86f279777c895c015ddff5cabc8f5a6dcd245d862a
+size 100697560

ckpts/ss_dec_conv3d_16l8_fp16.json

@@ -0,0 +1,12 @@
+
+{
+    "name": "SparseStructureDecoder",
+    "args": {
+        "out_channels": 1,
+        "latent_channels": 8,
+        "num_res_blocks": 2,
+        "num_res_blocks_middle": 2,
+        "channels": [512, 128, 32],
+        "use_fp16": true
+    }
+}

ckpts/ss_dec_conv3d_16l8_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c76d4a40519aa2d711cc263a8404105231ac26db31d946bed48b84fee79009a
+size 147591972

ckpts/ss_flow_img_dit_L_16l8_fp16.json

@@ -0,0 +1,17 @@
+{
+    "name": "SparseStructureFlowModel",
+    "args": {
+        "resolution": 16,
+        "in_channels": 8,
+        "out_channels": 8,
+        "model_channels": 1024,
+        "cond_channels": 1024,
+        "num_blocks": 24,
+        "num_heads": 16,
+        "mlp_ratio": 4,
+        "patch_size": 1,
+        "pe_mode": "ape",
+        "qk_rms_norm": true,
+        "use_fp16": true
+    }
+}

ckpts/ss_flow_img_dit_L_16l8_fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876df6751febc412d3e6f62c686f59da1634e4d3c4edba44fdf6a749bdbba73a
+size 1130770840

ckpts/ss_vggt_cond.json

@@ -0,0 +1,15 @@
+{
+    "name": "ModulatedMultiViewCond",
+    "args": {
+        "channels": 1024,
+        "ctx_channels": 3072,
+        "num_heads": 16,
+        "mlp_ratio": 4,
+        "attn_mode": "full",
+        "use_checkpoint": false,
+        "use_rope": false,
+        "share_mod": false,
+        "qk_rms_norm": true,
+        "qk_rms_norm_cross": false
+    }
+}

ckpts/ss_vggt_cond.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0fad4d17c8d6914c2320230ae24971ed6c6569a678e7d5c260f802e66895704
+size 352607808

pipeline.json

@@ -0,0 +1,61 @@
+{
+    "name": "TrellisVGGTTo3DPipeline",
+    "args": {
+        "models": {
+            "sparse_structure_decoder": "ckpts/ss_dec_conv3d_16l8_fp16",
+            "sparse_structure_flow_model": "ckpts/ss_flow_img_dit_L_16l8_fp16",
+            "slat_decoder_gs": "ckpts/slat_dec_gs_swin8_B_64l8gs32_fp16",
+            "slat_decoder_mesh": "ckpts/slat_dec_mesh_swin8_B_64l8m256c_fp16",
+            "slat_flow_model": "ckpts/slat_flow_img_dit_L_64l8p2_fp16",
+            "sparse_structure_vggt_cond": "ckpts/ss_vggt_cond",
+            "slat_vggt_cond": "ckpts/slat_vggt_cond"
+        },
+        "sparse_structure_sampler": {
+            "name": "FlowEulerGuidanceIntervalSampler",
+            "args": {
+                "sigma_min": 1e-5
+            },
+            "params": {
+                "steps": 25,
+                "cfg_strength": 5.0,
+                "cfg_interval": [0.5, 1.0],
+                "rescale_t": 3.0
+            }
+        },
+        "slat_sampler": {
+            "name": "FlowEulerGuidanceIntervalSampler",
+            "args": {
+                "sigma_min": 1e-5
+            },
+            "params": {
+                "steps": 25,
+                "cfg_strength": 5.0,
+                "cfg_interval": [0.5, 1.0],
+                "rescale_t": 3.0
+            }
+        },
+        "slat_normalization": {
+            "mean": [
+                -2.1687545776367188,
+                -0.004347046371549368,
+                -0.13352349400520325,
+                -0.08418072760105133,
+                -0.5271206498146057,
+                0.7238689064979553,
+                -1.1414450407028198,
+                1.2039363384246826
+            ],
+            "std": [
+                2.377650737762451,
+                2.386378288269043,
+                2.124418020248413,
+                2.1748552322387695,
+                2.663944721221924,
+                2.371192216873169,
+                2.6217446327209473,
+                2.684523105621338
+            ]
+        },
+        "image_cond_model": "dinov2_vitl14_reg"
+    }
+}