feat(models): publish the GenSfM inference bundle

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.cache/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 GenSfM authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSE_TRELLIS

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

README.md

@@ -0,0 +1,64 @@
+---
+language: en
+license: mit
+pipeline_tag: image-to-3d
+base_model: microsoft/TRELLIS-image-large
+tags:
+  - multi-view
+  - 3d-reconstruction
+  - camera-pose-estimation
+  - safetensors
+---
+
+# GenSfM pretrained models
+
+GenSfM jointly reconstructs an object and its cameras from two to four foreground-masked views. Four views are recommended. This repository contains the complete reconstruction weights for the native two-stage pipeline.
+
+## Usage
+
+Install the GenSfM code and its inference dependencies first. From the code repository, run the multi-view entrypoint:
+
+```bash
+python example_multi_image.py \
+  --pretrained dotblue/GenSfm \
+  --input-dir /path/to/object-views \
+  --output-dir outputs/my-object
+```
+
+The same argument also accepts a local model directory. Use `--revision` to select a specific Hub tag or commit.
+
+```python
+from trellis.pipelines import GenSfMPipeline
+
+pipeline = GenSfMPipeline.from_pretrained("dotblue/GenSfm").cuda()
+result = pipeline.run(["00.png", "01.png", "02.png", "03.png"], seed=1)
+mesh = result["mesh"][0]
+gaussians = result["gaussian"][0]
+extrinsics, intrinsics = result["extrinsics"], result["intrinsics"]
+```
+
+Inputs are ordered paths to RGBA images of the same rigid object. For photographs without alpha, add `--remove-background` to the CLI or pass `remove_background=True` to `run`. Camera intrinsics are normalized complete matrices, and extrinsics use OpenCV world-to-camera coordinates in the model's Z-up frame. The CLI exports a textured GLB, Gaussians, camera matrices, inputs and reprojections.
+
+DINOv2 (`dinov2_vitl14_reg`) is obtained separately through Torch Hub. Optional background removal uses rembg's own model. Both need a first-use download or an existing cache.
+
+## Contents
+
+`pipeline.json` maps six runtime roles to model prefixes and records both samplers, latent normalization and input preprocessing. Each prefix has a `.json` architecture file and a `.safetensors` state dictionary.
+
+| Prefix under `ckpts/` | Purpose |
+| --- | --- |
+| `stage1_joint` | Native joint shape and per-view camera flow model |
+| `stage2_slat` | Trained ConvNeXt geometry and appearance flow model |
+| `uv_decoder` | Trained UV representation decoder |
+| `ss_decoder` | TRELLIS occupancy decoder |
+| `slat_decoder_gs` | TRELLIS Gaussian decoder |
+| `slat_decoder_mesh` | TRELLIS mesh decoder |
+| `slat_encoder` | TRELLIS conditioning encoder loaded internally by stage 2 |
+
+No training directories or machine-specific paths are needed. The bundle includes inference weights; full training resume state is not included.
+
+## License and acknowledgements
+
+The model weights are released under the [MIT License](LICENSE).
+
+The sparse-structure decoder, SLat Gaussian/mesh decoders and SLat encoder originate from [microsoft/TRELLIS-image-large](https://huggingface.co/microsoft/TRELLIS-image-large). Their original MIT notice is preserved in `LICENSE_TRELLIS`. DINOv2 and optional background-removal weights are external dependencies and are not redistributed here.

ckpts/slat_decoder_gs.json

@@ -0,0 +1,31 @@
+{
+  "name": "SLatGaussianDecoder",
+  "args": {
+    "resolution": 64,
+    "model_channels": 768,
+    "latent_channels": 8,
+    "num_blocks": 12,
+    "num_heads": 12,
+    "mlp_ratio": 4,
+    "attn_mode": "swin",
+    "window_size": 8,
+    "use_fp16": true,
+    "representation_config": {
+      "lr": {
+        "_xyz": 1.0,
+        "_features_dc": 1.0,
+        "_opacity": 1.0,
+        "_scaling": 1.0,
+        "_rotation": 0.1
+      },
+      "perturb_offset": true,
+      "voxel_size": 1.5,
+      "num_gaussians": 32,
+      "2d_filter_kernel_size": 0.1,
+      "3d_filter_kernel_size": 0.0009,
+      "scaling_bias": 0.004,
+      "opacity_bias": 0.1,
+      "scaling_activation": "softplus"
+    }
+  }
+}

ckpts/slat_decoder_gs.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c84bcef5ce0af1f48b1b5558dabc7575a13346043c41a7e0610f1fa619a161
+size 171450952

ckpts/slat_decoder_mesh.json

@@ -0,0 +1,17 @@
+{
+  "name": "SLatMeshDecoder",
+  "args": {
+    "resolution": 64,
+    "model_channels": 768,
+    "latent_channels": 8,
+    "num_blocks": 12,
+    "num_heads": 12,
+    "mlp_ratio": 4,
+    "attn_mode": "swin",
+    "window_size": 8,
+    "use_fp16": true,
+    "representation_config": {
+      "use_color": true
+    }
+  }
+}

ckpts/slat_decoder_mesh.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e87aba94b5786407eb06d0502c1ed0885a0027a3f2b8537bfe15b0a92c01859
+size 181903412

ckpts/slat_encoder.json

@@ -0,0 +1,15 @@
+{
+  "name": "SLatEncoder",
+  "args": {
+    "resolution": 64,
+    "in_channels": 1024,
+    "model_channels": 768,
+    "latent_channels": 8,
+    "num_blocks": 12,
+    "num_heads": 12,
+    "mlp_ratio": 4,
+    "attn_mode": "swin",
+    "window_size": 8,
+    "use_fp16": true
+  }
+}

ckpts/slat_encoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21dceac6bee917ab6458ff52c9757ba89a779d03031c7bd17f9e7f0103bfd436
+size 173242816

ckpts/ss_decoder.json

@@ -0,0 +1,15 @@
+{
+  "name": "SparseStructureDecoder",
+  "args": {
+    "out_channels": 1,
+    "latent_channels": 8,
+    "num_res_blocks": 2,
+    "num_res_blocks_middle": 2,
+    "channels": [
+      512,
+      128,
+      32
+    ],
+    "use_fp16": true
+  }
+}

ckpts/ss_decoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c76d4a40519aa2d711cc263a8404105231ac26db31d946bed48b84fee79009a
+size 147591972

ckpts/stage1_joint.json

@@ -0,0 +1,37 @@
+{
+  "name": "VisualMultiPoseSparseStructureFlowModel",
+  "args": {
+    "resolution": 16,
+    "in_channels": 8,
+    "out_channels": 8,
+    "pose_resolution": 4,
+    "pose_in_channels": 8,
+    "pose_out_channels": 8,
+    "model_channels": 1024,
+    "cond_channels": 1536,
+    "dino_cond_channels": 1024,
+    "visual_input_channels": 3,
+    "visual_feat_resolution": 296,
+    "visual_patch_resolution": 37,
+    "visual_feat_channels": [
+      6,
+      6,
+      6,
+      8
+    ],
+    "visual_conv_kernel_size": 5,
+    "num_blocks": 24,
+    "num_heads": 16,
+    "mlp_ratio": 4,
+    "patch_size": 1,
+    "pose_patch_size": 1,
+    "pe_mode": "ape",
+    "fusion_layer_type": null,
+    "self_attn_layout": "joint",
+    "cross_attn_layout": "full",
+    "num_view_embeds": 4,
+    "num_shared_register_tokens": 128,
+    "qk_rms_norm": true,
+    "use_fp16": true
+  }
+}

ckpts/stage1_joint.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a31f03d184af60c14db71e3799397744bb53ae3d4b520c229b8d45bed83ec1c9
+size 2340577920

ckpts/stage2_slat.json

@@ -0,0 +1,38 @@
+{
+  "name": "ConvNeXtVisualLatentConditioningSLatFlowModel",
+  "args": {
+    "resolution": 64,
+    "in_channels": 8,
+    "out_channels": 8,
+    "model_channels": 1024,
+    "cond_channels": 1024,
+    "cond_latent_channels": 8,
+    "visual_feat_channels": [
+      4,
+      4,
+      4,
+      4
+    ],
+    "visual_feat_resolution": 256,
+    "hidden_channels": 32,
+    "num_pyramid_blocks": 1,
+    "pyramid_mlp_ratio": 4.0,
+    "dwconv_kernel_size": 7,
+    "stem_kernel_size": 3,
+    "stem_stride": 1,
+    "layer_scale_init": 1e-06,
+    "mask_injection": "stem",
+    "num_blocks": 24,
+    "num_heads": 16,
+    "mlp_ratio": 4,
+    "patch_size": 2,
+    "num_io_res_blocks": 2,
+    "io_block_channels": [
+      128
+    ],
+    "pe_mode": "ape",
+    "qk_rms_norm": true,
+    "use_fp16": true,
+    "pretrained_slat_enc": "./slat_encoder"
+  }
+}

ckpts/stage2_slat.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f4bd0ec513eb9727dd7240eff34c68c8749d7e9d97a7640e0703fa2e391c024
+size 2402022552

ckpts/uv_decoder.json

@@ -0,0 +1,17 @@
+{
+  "name": "SparseStructureDecoder",
+  "args": {
+    "out_channels": 3,
+    "latent_channels": 8,
+    "num_res_blocks": 2,
+    "num_res_blocks_middle": 2,
+    "channels": [
+      512,
+      256,
+      128,
+      64,
+      32
+    ],
+    "use_fp16": true
+  }
+}

ckpts/uv_decoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc93c8fb1b4a3d52c92784bc5733fe905ec886515c5c74651546e8a0f9a14db8
+size 415051644

pipeline.json

@@ -0,0 +1,85 @@
+{
+  "name": "GenSfMPipeline",
+  "args": {
+    "models": {
+      "denoiser": "ckpts/stage1_joint",
+      "slat_denoiser": "ckpts/stage2_slat",
+      "ss_decoder": "ckpts/ss_decoder",
+      "suv_decoder": "ckpts/uv_decoder",
+      "slat_decoder_gs": "ckpts/slat_decoder_gs",
+      "slat_decoder_mesh": "ckpts/slat_decoder_mesh"
+    },
+    "inference": {
+      "normalization": {
+        "mean": [
+          -2.1687545776367188,
+          -0.004347046371549368,
+          -0.13352349400520325,
+          -0.08418072760105133,
+          -0.5271206498146057,
+          0.7238689064979553,
+          -1.1414450407028198,
+          1.2039363384246826
+        ],
+        "std": [
+          2.377650737762451,
+          2.386378288269043,
+          2.124418020248413,
+          2.1748552322387695,
+          2.663944721221924,
+          2.371192216873169,
+          2.6217446327209473,
+          2.684523105621338
+        ]
+      },
+      "ssuv_sampler_args": {
+        "name": "MultiPoseFlowEulerGuidanceIntervalSampler",
+        "args": {
+          "sigma_min": 1e-05
+        },
+        "params": {
+          "steps": 25,
+          "cfg_strength": 7.5,
+          "cfg_interval": [
+            0.5,
+            1.0
+          ],
+          "rescale_t": 3.0
+        }
+      },
+      "slat_sampler_args": {
+        "name": "FlowEulerGuidanceIntervalSampler",
+        "args": {
+          "sigma_min": 1e-05
+        },
+        "params": {
+          "steps": 25,
+          "cfg_strength": 3.0,
+          "cfg_interval": [
+            0.5,
+            1.0
+          ],
+          "rescale_t": 3.0
+        }
+      },
+      "image_cond_model": "dinov2_vitl14_reg",
+      "ssuv_uses_visual": true,
+      "predict_ssuv_from_ss": true,
+      "dilation_sigma": 0.5,
+      "dilation_min_weight": 0.05,
+      "render_resolution": 518,
+      "render_bg_color": [
+        0.0,
+        0.0,
+        0.0
+      ],
+      "ssuv_expand_min_num_activated": 10,
+      "ssuv_expand_threshold_step": 0.05
+    },
+    "preprocessing": {
+      "image_size": 518,
+      "crop_size_ratio": 1.2,
+      "max_views": 4
+    }
+  }
+}