Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/10_cyrillic_newspaper_press_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/nvidia_example_caption_bf16.png filter=lfs diff=lfs merge=lfs -text
+examples/nvidia_example_caption_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/01_metro_archive_reading_room_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/02_arctic_greenhouse_night_shift_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/03_control_room_restoration_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/04_rain_market_cross_section_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/05_manuscript_restoration_table_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/06_robotic_assembly_line_signage_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/07_kitchen_storm_chess_table_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/08_orbital_cockpit_cyrillic_ui_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text
+examples/09_flood_command_center_modelopt_fp8.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: nvidia/Cosmos3-Super-Text2Image
+library_name: diffusers
+pipeline_tag: text-to-image
+tags:
+  - cosmos3
+  - diffusers
+  - modelopt
+  - fp8
+  - nvidia
+  - text-to-image
+license: other
+license_name: openmdw1.1-license
+license_link: https://openmdw.ai/license/1-1/
+---
+
+# Cosmos3-Super-Text2Image NVIDIA ModelOpt FP8 Transformer
+
+This repository contains a transformer-only NVIDIA ModelOpt FP8 quantization for [nvidia/Cosmos3-Super-Text2Image](https://huggingface.co/nvidia/Cosmos3-Super-Text2Image).
+
+It does not repeat the original model card. Read NVIDIA's model card, prompt-format guidance, license, and safety notes here:
+[nvidia/Cosmos3-Super-Text2Image](https://huggingface.co/nvidia/Cosmos3-Super-Text2Image).
+
+Only `transformer/` is provided as a weight artifact. The VAE, scheduler, tokenizers, safety checker, and other components are loaded from the base model.
+
+## Recipe
+
+| Setting | Value |
+| --- | --- |
+| Quantizer | NVIDIA ModelOpt |
+| ModelOpt version | `0.44.0` |
+| Quant type | `FP8_DEFAULT_CFG` |
+| Weight-only | `True` |
+| Compressed | `True` |
+| Quantized modules inserted | `2709` |
+| Quantization time | 1.34s |
+| Compress time | 0.45s |
+| Save time | 65.99s |
+| Transformer checkpoint size | 61.06 GiB |
+
+The checkpoint includes ModelOpt state in `transformer/modelopt_state.pth`.
+
+## Assemble The Pipeline
+
+Install ModelOpt in the same environment as Diffusers:
+
+```bash
+pip install "nvidia_modelopt[hf]"
+```
+
+The current tested runtime requires a small compatibility helper for ModelOpt `QTensorWrapper` restoration with Diffusers and Accelerate. Important: load the quantized transformer **without** passing `torch_dtype`; otherwise Diffusers casts FP8 tensors back to BF16 during state-dict loading.
+
+```python
+import json
+import torch
+from diffusers import Cosmos3OmniPipeline, Cosmos3OmniTransformer
+from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler
+from modelopt.torch.quantization.qtensor.base_qtensor import QTensorWrapper
+import modelopt.torch.opt as mto
+
+
+def patch_modelopt_qtensor_loader():
+    import accelerate.utils.modeling as accelerate_modeling
+    import diffusers.models.model_loading_utils as diffusers_loading
+
+    original = accelerate_modeling.set_module_tensor_to_device
+    if getattr(original, "_cosmos3_modelopt_patch", False):
+        return
+
+    def patched(module, tensor_name, device, value=None, dtype=None, fp16_statistics=None,
+                tied_params_map=None, non_blocking=False, clear_cache=True):
+        leaf_module = module
+        leaf_name = tensor_name
+        if "." in tensor_name:
+            parts = tensor_name.split(".")
+            for part in parts[:-1]:
+                leaf_module = getattr(leaf_module, part)
+            leaf_name = parts[-1]
+        old_value = getattr(leaf_module, leaf_name) if hasattr(leaf_module, leaf_name) else None
+        if isinstance(old_value, QTensorWrapper) and value is not None:
+            leaf_module._parameters[leaf_name] = QTensorWrapper(
+                value.to(device, non_blocking=non_blocking),
+                metadata=old_value.metadata,
+            )
+            return
+        return original(module, tensor_name, device, value, dtype, fp16_statistics,
+                        tied_params_map, non_blocking, clear_cache)
+
+    patched._cosmos3_modelopt_patch = True
+    accelerate_modeling.set_module_tensor_to_device = patched
+    diffusers_loading.set_module_tensor_to_device = patched
+
+
+def cast_modelopt_runtime_tensors(model, dtype=torch.bfloat16):
+    for module in model.modules():
+        for name, param in list(module._parameters.items()):
+            if isinstance(param, QTensorWrapper):
+                param.metadata["dtype"] = dtype
+            elif param is not None and param.is_floating_point():
+                module._parameters[name] = torch.nn.Parameter(
+                    param.detach().to(dtype),
+                    requires_grad=param.requires_grad,
+                )
+        for name, buf in list(module._buffers.items()):
+            if buf is not None and buf.is_floating_point():
+                module._buffers[name] = buf.to(dtype)
+    return model
+
+
+patch_modelopt_qtensor_loader()
+mto.enable_huggingface_checkpointing()
+
+transformer = Cosmos3OmniTransformer.from_pretrained(
+    "WaveCut/Cosmos3-Super-Text2Image-ModelOpt-FP8-Transformer",
+    subfolder="transformer",
+    use_safetensors=False,
+)
+transformer = cast_modelopt_runtime_tensors(transformer, torch.bfloat16)
+
+pipe = Cosmos3OmniPipeline.from_pretrained(
+    "nvidia/Cosmos3-Super-Text2Image",
+    transformer=transformer,
+    torch_dtype=torch.bfloat16,
+    device_map="cuda",
+    enable_safety_checker=True,
+)
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=3.0)
+pipe.to("cuda")
+
+json_caption = {
+    "subjects": [],
+    "background_setting": "A concise scene description.",
+    "comprehensive_t2i_caption": "A detailed natural-language caption.",
+    "resolution": {"H": 1024, "W": 1024},
+    "aspect_ratio": "1,1",
+}
+
+with torch.autocast("cuda", dtype=torch.bfloat16):
+    result = pipe(
+        prompt=json.dumps(json_caption),
+        negative_prompt="",
+        num_frames=1,
+        height=1024,
+        width=1024,
+        num_inference_steps=50,
+        guidance_scale=4.0,
+        generator=torch.Generator(device="cuda").manual_seed(1143),
+    )
+result.video[0].save("cosmos3_modelopt_fp8.png")
+```
+
+## Benchmarks
+
+Measured on one RunPod NVIDIA B200 instance with local container storage, cached model files, PyTorch `2.9.1+cu130`, 1024x1024 image generation, 50 inference steps, guidance scale 4.0, `flow_shift=3.0`, system prompt enabled. The ModelOpt FP8 runtime uses BF16 autocast around the pipeline forward.
+
+### Transformer Component Load
+
+| Variant | Load to CUDA | VRAM after load | Torch allocated | Torch reserved | Transformer weights |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| BF16 base transformer | 41.83s | 122,758 MiB | 122,121 MiB | 122,132 MiB | 119.21 GiB |
+| NVIDIA ModelOpt FP8 transformer | 21.95s | 63,550 MiB | 62,907 MiB | 62,924 MiB | 61.06 GiB |
+
+### Full Pipeline Generation
+
+The stress set is ten handwritten JSON-caption prompts designed to stress Cyrillic text, reflections, multi-object composition, anatomy, small details, and scene-following.
+
+| Variant | Full pipeline load | VRAM after load | Torch allocated after load | Avg generation time | Min / max generation time | Peak sampled VRAM | Images |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| BF16 base pipeline | 31.31s | 125,134 MiB | 124,386 MiB | 16.05s | 15.51s / 17.97s | 141,104 MiB | 10 |
+| NVIDIA ModelOpt FP8 pipeline | 35.49s | 65,810 MiB | 65,171 MiB | 45.57s | 45.07s / 47.28s | 81,854 MiB | 10 |
+
+### Original NVIDIA Example Caption
+
+The original model repository provides [`assets/example_caption.json`](https://huggingface.co/nvidia/Cosmos3-Super-Text2Image/blob/main/assets/example_caption.json). The images below are generated locally with the same JSON-caption, seed 1143, 1024x1024, 50 steps, guidance scale 4.0.
+
+| Variant | Pipeline load | Generation time | Peak sampled VRAM |
+| --- | ---: | ---: | ---: |
+| BF16 base pipeline | 35.41s | 18.01s | 141,098 MiB |
+| NVIDIA ModelOpt FP8 pipeline | 35.28s | 47.20s | 71,470 MiB |
+
+BF16 reference output:
+
+![BF16 output for NVIDIA example caption](examples/nvidia_example_caption_bf16.png)
+
+NVIDIA ModelOpt FP8 output:
+
+![NVIDIA ModelOpt FP8 output for NVIDIA example caption](examples/nvidia_example_caption_modelopt_fp8.png)
+
+## Stress Prompt Outputs
+
+| Stress prompt | NVIDIA ModelOpt FP8 output |
+| --- | --- |
+| 01 metro archive reading room | ![01 metro archive reading room](examples/01_metro_archive_reading_room_modelopt_fp8.png) |
+| 02 arctic greenhouse night shift | ![02 arctic greenhouse night shift](examples/02_arctic_greenhouse_night_shift_modelopt_fp8.png) |
+| 03 control room restoration | ![03 control room restoration](examples/03_control_room_restoration_modelopt_fp8.png) |
+| 04 rain market cross section | ![04 rain market cross section](examples/04_rain_market_cross_section_modelopt_fp8.png) |
+| 05 manuscript restoration table | ![05 manuscript restoration table](examples/05_manuscript_restoration_table_modelopt_fp8.png) |
+| 06 robotic assembly line signage | ![06 robotic assembly line signage](examples/06_robotic_assembly_line_signage_modelopt_fp8.png) |
+| 07 kitchen storm chess table | ![07 kitchen storm chess table](examples/07_kitchen_storm_chess_table_modelopt_fp8.png) |
+| 08 orbital cockpit cyrillic ui | ![08 orbital cockpit cyrillic ui](examples/08_orbital_cockpit_cyrillic_ui_modelopt_fp8.png) |
+| 09 flood command center | ![09 flood command center](examples/09_flood_command_center_modelopt_fp8.png) |
+| 10 cyrillic newspaper press | ![10 cyrillic newspaper press](examples/10_cyrillic_newspaper_press_modelopt_fp8.png) |
+
+## Notes
+
+- Treat this as an experimental ModelOpt FP8 transformer artifact. The upstream NVIDIA card documents BF16 as the tested precision.
+- Do not pass `torch_dtype=torch.bfloat16` when loading this quantized transformer; cast runtime metadata after loading as shown above.
+- The safety checker is not included in this repository; load it from the base model if your use case requires it.
+- Text rendering, especially exact Cyrillic text, remains a hard case for this model family and should be evaluated visually for the target prompt distribution.

transformer/config.json

@@ -0,0 +1,60 @@
+{
+  "_class_name": "Cosmos3OmniTransformer",
+  "_diffusers_version": "0.39.0.dev0",
+  "_name_or_path": "nvidia/Cosmos3-Super-Text2Image",
+  "action_dim": 32,
+  "action_gen": false,
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "base_fps": 24,
+  "dtype": "bfloat16",
+  "enable_fps_modulation": true,
+  "freeze_und": false,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 25600,
+  "joint_attn_implementation": "two_way",
+  "latent_channel": 48,
+  "latent_patch_size": 2,
+  "max_action_dim": 32,
+  "max_position_embeddings": 262144,
+  "model_type": "qwen3_vl_text",
+  "num_attention_heads": 64,
+  "num_embodiment_domains": 32,
+  "num_hidden_layers": 64,
+  "num_key_value_heads": 8,
+  "patch_latent_dim": 192,
+  "position_embedding_type": "unified_3d_mrope",
+  "qk_norm": false,
+  "qk_norm_for_diffusion": true,
+  "qk_norm_for_text": true,
+  "rms_norm_eps": 1e-06,
+  "rope_axes_dim": [
+    24,
+    20,
+    20
+  ],
+  "rope_scaling": {
+    "mrope_interleaved": true,
+    "mrope_section": [
+      24,
+      20,
+      20
+    ],
+    "rope_type": "default"
+  },
+  "rope_theta": 5000000,
+  "sound_dim": 64,
+  "sound_gen": true,
+  "sound_latent_fps": 25,
+  "temporal_compression_factor_sound": 1,
+  "timestep_scale": 0.001,
+  "unified_3d_mrope_reset_spatial_ids": true,
+  "unified_3d_mrope_temporal_modality_margin": 15000,
+  "use_cache": true,
+  "use_moe": true,
+  "video_temporal_causal": false,
+  "vocab_size": 151936
+}

transformer/diffusion_pytorch_model-00001-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6e3b47a834bef64dc72502f59a5099941fb61aeade649696a0428a2f24fe982
+size 4932444015

transformer/diffusion_pytorch_model-00002-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf167e911627d35d7450e2786602d3971428ad53b1dbf052a314f3f06afe817a
+size 4876178507

transformer/diffusion_pytorch_model-00003-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d7acc514c7e9bfb5c10247883d1ec84a3b7b2dc52be834ee34768a278512d04
+size 4876178699

transformer/diffusion_pytorch_model-00004-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97a0bef6cbb63a5c4eac1ee5e659c07b8aa9d7af216bd6c24f194997dd31f634
+size 4876178763

transformer/diffusion_pytorch_model-00005-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:613e18d6f91c114a4c3e4d3b847fa2e54d8f1ba04ff72127408ec4a57b217f85
+size 4876178763

transformer/diffusion_pytorch_model-00006-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a63f21d31cc1fbe26f2fd54c7b154f27ffd2e8580136fb405fe903451cf2b582
+size 4876178763

transformer/diffusion_pytorch_model-00007-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c73330ef10b69e1959b6e30610dedf543ffdc8cda5179ed92fc5bc2231f3de6
+size 4876178763

transformer/diffusion_pytorch_model-00008-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ab859b1361b3d263e8e43b6b528fc023bac7ad5069edf795ea393b8b0d7e769
+size 4876178763

transformer/diffusion_pytorch_model-00009-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e2e4b5b592adf30fbe5d0b87f409b6fd68bf67848519797e562b57637ec4ffa
+size 4876178763

transformer/diffusion_pytorch_model-00010-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a703609490a65ba8e34175cc2045fb9ca5fac5904c7cc77f3b059f482c56ddce
+size 4876178763

transformer/diffusion_pytorch_model-00011-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0cba2114cbcfd28e836baec3d9afaa4ff856f15bb6683dfe726105442ee96f7
+size 4876178763

transformer/diffusion_pytorch_model-00012-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b72f854e65f6983114e5ce7ee0802c5efeb388df754273b916bbb61700f429da
+size 4876178763

transformer/diffusion_pytorch_model-00013-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d12211b95096a739b274bc321ebb315ac05429493bf6000cc308a6fc92179eef
+size 4876178763

transformer/diffusion_pytorch_model-00014-of-00014.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c12ea11e672fdfc91724f6ad5297f56dbb95410c416e3df9a90f9a4dda57fe71
+size 2111707809

transformer/modelopt_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92c2d7ff49a4c8bcafddf083239bfd48a833d43e21855fa1f113e7c8521373a1
+size 1242391