Add H100 skill 5s validation artifacts

.gitattributes

@@ -34,3 +34,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 validation/h100_20260420/artifacts/hunyuanvideo_bf16_fp8_contact_sheet.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/contact_frames/bf16_0.5s.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/contact_frames/bf16_2.5s.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/contact_frames/bf16_4.5s.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/contact_frames/fp8_0.5s.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/contact_frames/fp8_2.5s.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/contact_frames/fp8_4.5s.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/hunyuanvideo_bf16_fp8_skill_5s_contact_sheet.png filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/hunyuanvideo_bf16_skill_5s.mp4 filter=lfs diff=lfs merge=lfs -text
+validation/h100_skill_5s_20260420/artifacts/hunyuanvideo_fp8_skill_5s.mp4 filter=lfs diff=lfs merge=lfs -text

validation/h100_skill_5s_20260420/artifacts/hunyuanvideo_bf16_skill_5s.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20ccf74638167be65a1ce5d668e4d596b57987cd6f4b3624c085d5eb12b04e88
+size 183737

validation/h100_skill_5s_20260420/artifacts/hunyuanvideo_fp8_skill_5s.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a566756b3b4d996a78054d539c5279a23d6d34ccb62ab014a3e7ecd49b01eaf6
+size 152026

validation/h100_skill_5s_20260420/artifacts/video_probe_skill_5s.json

@@ -0,0 +1,18 @@
+{
+  "BF16": {
+    "width": 848,
+    "height": 480,
+    "r_frame_rate": "13/1",
+    "avg_frame_rate": "13/1",
+    "duration": "5.000000",
+    "nb_frames": "65"
+  },
+  "FP8": {
+    "width": 848,
+    "height": 480,
+    "r_frame_rate": "13/1",
+    "avg_frame_rate": "13/1",
+    "duration": "5.000000",
+    "nb_frames": "65"
+  }
+}

validation/h100_skill_5s_20260420/benchmark/compare_perf_bf16_vs_fp8.txt

@@ -0,0 +1,27 @@
+### Performance Comparison Report
+
+#### 1. High-level Summary
+| Metric | Baseline | New | Diff | Status |
+| :--- | :--- | :--- | :--- | :--- |
+| **E2E Latency** | 59545.91 ms | 54747.61 ms | **-4798.30 ms (-8.1%)** | ✅ |
+| **Throughput** | 0.02 req/s | 0.02 req/s | - | - |
+
+
+#### 2. Stage Breakdown
+| Stage Name | Baseline (ms) | New (ms) | Diff (ms) | Diff (%) | Status |
+| :--- | :--- | :--- | :--- | :--- | :--- |
+| InputValidationStage | 0.04 | 0.03 | -0.01 | -17.9% | ⚪️ |
+| TextEncodingStage | 308.48 | 306.44 | -2.04 | -0.7% | ⚪️ |
+| TimestepPreparationStage | 0.41 | 0.34 | -0.07 | -17.3% | ⚪️ |
+| LatentPreparationStage | 0.14 | 0.12 | -0.02 | -14.3% | ⚪️ |
+| DenoisingStage | 42542.10 | 37979.67 | -4562.43 | -10.7% | 🟢 |
+| DecodingStage | 16691.68 | 16458.20 | -233.47 | -1.4% | ⚪️ |
+
+
+<details>
+<summary>Metadata</summary>
+
+- Baseline Commit: `N/A`
+- New Commit: `N/A`
+- Timestamp: 2026-04-20T03:07:09.034563
+</details>

validation/h100_skill_5s_20260420/benchmark/hunyuanvideo_bf16_skill_5s_perf.json

@@ -0,0 +1,175 @@
+{
+  "timestamp": "2026-04-20T03:04:12.058611+00:00",
+  "request_id": "5c3703ab-db84-46b8-ba90-a180b6dcc5f1",
+  "commit_hash": "N/A",
+  "tag": "cli_generate",
+  "total_duration_ms": 59545.909993117675,
+  "steps": [
+    {
+      "name": "InputValidationStage",
+      "duration_ms": 0.04043290391564369
+    },
+    {
+      "name": "TextEncodingStage",
+      "duration_ms": 308.477042010054
+    },
+    {
+      "name": "TimestepPreparationStage",
+      "duration_ms": 0.4082408268004656
+    },
+    {
+      "name": "LatentPreparationStage",
+      "duration_ms": 0.14178105629980564
+    },
+    {
+      "name": "DenoisingStage",
+      "duration_ms": 42542.099812068045
+    },
+    {
+      "name": "DecodingStage",
+      "duration_ms": 16691.675554960966
+    }
+  ],
+  "denoise_steps_ms": [
+    {
+      "step": 0,
+      "duration_ms": 261.8462292011827
+    },
+    {
+      "step": 1,
+      "duration_ms": 1436.7910819128156
+    },
+    {
+      "step": 2,
+      "duration_ms": 1469.7598610073328
+    },
+    {
+      "step": 3,
+      "duration_ms": 1440.979610895738
+    },
+    {
+      "step": 4,
+      "duration_ms": 1464.3825711682439
+    },
+    {
+      "step": 5,
+      "duration_ms": 1447.7491180878133
+    },
+    {
+      "step": 6,
+      "duration_ms": 1452.3683309089392
+    },
+    {
+      "step": 7,
+      "duration_ms": 1452.8397121466696
+    },
+    {
+      "step": 8,
+      "duration_ms": 1456.2478899024427
+    },
+    {
+      "step": 9,
+      "duration_ms": 1453.0877869110554
+    },
+    {
+      "step": 10,
+      "duration_ms": 1456.1456739902496
+    },
+    {
+      "step": 11,
+      "duration_ms": 1456.114053959027
+    },
+    {
+      "step": 12,
+      "duration_ms": 1457.9589350614697
+    },
+    {
+      "step": 13,
+      "duration_ms": 1456.3369450625032
+    },
+    {
+      "step": 14,
+      "duration_ms": 1458.0988800153136
+    },
+    {
+      "step": 15,
+      "duration_ms": 1460.4044100269675
+    },
+    {
+      "step": 16,
+      "duration_ms": 1457.6487878803164
+    },
+    {
+      "step": 17,
+      "duration_ms": 1465.2510930318385
+    },
+    {
+      "step": 18,
+      "duration_ms": 1459.1034480836242
+    },
+    {
+      "step": 19,
+      "duration_ms": 1461.6714399307966
+    },
+    {
+      "step": 20,
+      "duration_ms": 1460.5804041493684
+    },
+    {
+      "step": 21,
+      "duration_ms": 1461.9483698625118
+    },
+    {
+      "step": 22,
+      "duration_ms": 1461.8699769489467
+    },
+    {
+      "step": 23,
+      "duration_ms": 1462.3029250651598
+    },
+    {
+      "step": 24,
+      "duration_ms": 1461.5925929974765
+    },
+    {
+      "step": 25,
+      "duration_ms": 1458.9864378795028
+    },
+    {
+      "step": 26,
+      "duration_ms": 1463.2249271962792
+    },
+    {
+      "step": 27,
+      "duration_ms": 1463.981956942007
+    },
+    {
+      "step": 28,
+      "duration_ms": 1461.630366044119
+    },
+    {
+      "step": 29,
+      "duration_ms": 1457.109339069575
+    }
+  ],
+  "memory_checkpoints": {
+    "before_forward": {
+      "allocated_mb": 24920.08,
+      "reserved_mb": 48128.0,
+      "peak_allocated_mb": 24920.08,
+      "peak_reserved_mb": 48128.0
+    },
+    "after_forward": {
+      "allocated_mb": 24796.44,
+      "reserved_mb": 48128.0,
+      "peak_allocated_mb": 44609.86,
+      "peak_reserved_mb": 48128.0
+    }
+  },
+  "meta": {
+    "prompt": [
+      "A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window."
+    ],
+    "model": "/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773"
+  }
+}

validation/h100_skill_5s_20260420/benchmark/hunyuanvideo_fp8_skill_5s_perf.json

@@ -0,0 +1,175 @@
+{
+  "timestamp": "2026-04-20T03:06:47.870941+00:00",
+  "request_id": "ac2d8c62-5d6c-4ef6-a0fb-b71654e9e2f8",
+  "commit_hash": "N/A",
+  "tag": "cli_generate",
+  "total_duration_ms": 54747.61236994527,
+  "steps": [
+    {
+      "name": "InputValidationStage",
+      "duration_ms": 0.03319093957543373
+    },
+    {
+      "name": "TextEncodingStage",
+      "duration_ms": 306.4362029545009
+    },
+    {
+      "name": "TimestepPreparationStage",
+      "duration_ms": 0.33745612017810345
+    },
+    {
+      "name": "LatentPreparationStage",
+      "duration_ms": 0.12155994772911072
+    },
+    {
+      "name": "DenoisingStage",
+      "duration_ms": 37979.66786590405
+    },
+    {
+      "name": "DecodingStage",
+      "duration_ms": 16458.200897090137
+    }
+  ],
+  "denoise_steps_ms": [
+    {
+      "step": 0,
+      "duration_ms": 256.218392169103
+    },
+    {
+      "step": 1,
+      "duration_ms": 1285.6723698787391
+    },
+    {
+      "step": 2,
+      "duration_ms": 1290.6845030374825
+    },
+    {
+      "step": 3,
+      "duration_ms": 1301.6360928304493
+    },
+    {
+      "step": 4,
+      "duration_ms": 1297.4275317974389
+    },
+    {
+      "step": 5,
+      "duration_ms": 1296.6165849938989
+    },
+    {
+      "step": 6,
+      "duration_ms": 1303.3105719368905
+    },
+    {
+      "step": 7,
+      "duration_ms": 1303.848479874432
+    },
+    {
+      "step": 8,
+      "duration_ms": 1299.7515010647476
+    },
+    {
+      "step": 9,
+      "duration_ms": 1302.7811218053102
+    },
+    {
+      "step": 10,
+      "duration_ms": 1301.1772891040891
+    },
+    {
+      "step": 11,
+      "duration_ms": 1300.531161017716
+    },
+    {
+      "step": 12,
+      "duration_ms": 1298.6012869514525
+    },
+    {
+      "step": 13,
+      "duration_ms": 1297.2778889816254
+    },
+    {
+      "step": 14,
+      "duration_ms": 1299.553635995835
+    },
+    {
+      "step": 15,
+      "duration_ms": 1302.214392926544
+    },
+    {
+      "step": 16,
+      "duration_ms": 1301.3874609023333
+    },
+    {
+      "step": 17,
+      "duration_ms": 1300.045948009938
+    },
+    {
+      "step": 18,
+      "duration_ms": 1300.40623084642
+    },
+    {
+      "step": 19,
+      "duration_ms": 1303.7210528273135
+    },
+    {
+      "step": 20,
+      "duration_ms": 1303.497269982472
+    },
+    {
+      "step": 21,
+      "duration_ms": 1303.9344048593193
+    },
+    {
+      "step": 22,
+      "duration_ms": 1300.4847038537264
+    },
+    {
+      "step": 23,
+      "duration_ms": 1302.0818289369345
+    },
+    {
+      "step": 24,
+      "duration_ms": 1303.708059946075
+    },
+    {
+      "step": 25,
+      "duration_ms": 1307.097784942016
+    },
+    {
+      "step": 26,
+      "duration_ms": 1305.4667548276484
+    },
+    {
+      "step": 27,
+      "duration_ms": 1304.4850858859718
+    },
+    {
+      "step": 28,
+      "duration_ms": 1301.8505610525608
+    },
+    {
+      "step": 29,
+      "duration_ms": 1301.1197270825505
+    }
+  ],
+  "memory_checkpoints": {
+    "before_forward": {
+      "allocated_mb": 16370.1,
+      "reserved_mb": 38872.0,
+      "peak_allocated_mb": 16370.1,
+      "peak_reserved_mb": 38872.0
+    },
+    "after_forward": {
+      "allocated_mb": 16246.53,
+      "reserved_mb": 38872.0,
+      "peak_allocated_mb": 36059.77,
+      "peak_reserved_mb": 38872.0
+    }
+  },
+  "meta": {
+    "prompt": [
+      "A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window."
+    ],
+    "model": "/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773"
+  }
+}

validation/h100_skill_5s_20260420/logs/generate_bf16_skill_5s.log

@@ -0,0 +1,98 @@
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:01:48] Warmup enabled, the launch time is expected to be longer than usual
+[04-20 03:01:48] server_args: {"model_path": "/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773", "model_id": null, "backend": "sglang", "attention_backend": null, "attention_backend_config": {}, "cache_dit_config": null, "nccl_port": null, "trust_remote_code": false, "revision": null, "num_gpus": 1, "tp_size": 1, "sp_degree": 1, "ulysses_degree": 1, "ring_degree": 1, "dp_size": 1, "dp_degree": 1, "enable_cfg_parallel": false, "hsdp_replicate_dim": 1, "hsdp_shard_dim": 1, "dist_timeout": 3600, "pipeline_class_name": null, "lora_path": null, "lora_nickname": "default", "lora_scale": 1.0, "lora_weight_name": null, "component_paths": {}, "transformer_weights_path": null, "lora_target_modules": null, "dit_cpu_offload": true, "dit_layerwise_offload": null, "dit_offload_prefetch_size": 0.0, "text_encoder_cpu_offload": true, "image_encoder_cpu_offload": true, "vae_cpu_offload": true, "use_fsdp_inference": false, "pin_cpu_memory": true, "ltx2_two_stage_device_mode": null, "comfyui_mode": false, "enable_torch_compile": true, "warmup": true, "warmup_resolutions": null, "warmup_steps": 1, "disable_autocast": false, "master_port": 30005, "host": "127.0.0.1", "port": 30000, "webui": false, "webui_port": 12312, "scheduler_port": 5589, "strict_ports": false, "output_path": "/data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts", "input_save_path": "inputs/uploads", "prompt_file_path": null, "model_paths": {}, "model_loaded": {"transformer": true, "vae": true, "video_vae": true, "audio_vae": true, "video_dit": true, "audio_dit": true, "dual_tower_bridge": true}, "boundary_ratio": null, "base_gpu_id": 0, "disagg_role": "monolithic", "disagg_timeout": 600, "disagg_dispatch_policy": "round_robin", "disagg_mode": false, "disagg_server_addr": null, "encoder_urls": null, "denoiser_urls": null, "decoder_urls": null, "encoder_tp": null, "denoiser_tp": null, "denoiser_sp": null, "denoiser_ulysses": null, "denoiser_ring": null, "decoder_tp": null, "disagg_transfer_pool_size": 268435456, "disagg_p2p_hostname": "127.0.0.1", "disagg_ib_device": null, "pool_work_endpoint": null, "pool_result_endpoint": null, "log_level": "info", "uvicorn_access_log_exclude_prefixes": []}
+[04-20 03:01:48] Local mode: True
+[04-20 03:01:48] Starting server...
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:01:55] Scheduler bind at endpoint: tcp://127.0.0.1:5589
+[04-20 03:01:55] Initializing distributed environment with world_size=1, device=cuda:0, timeout=3600
+[04-20 03:01:55] Setting distributed timeout to 3600 seconds
+[04-20 03:01:56] No pipeline_class_name specified, using model_index.json
+[04-20 03:01:56] Diffusers version: 0.32.0.dev0
+[04-20 03:01:56] Using pipeline from model_index.json: HunyuanVideoPipeline
+[04-20 03:01:56] Loading pipeline modules...
+[04-20 03:01:56] Model already exists locally and is complete
+[04-20 03:01:56] Model path: /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773
+[04-20 03:01:56] Diffusers version: 0.32.0.dev0
+[04-20 03:01:56] Loading pipeline modules from config: {'_class_name': 'HunyuanVideoPipeline', '_diffusers_version': '0.32.0.dev0', 'scheduler': ['diffusers', 'FlowMatchEulerDiscreteScheduler'], 'text_encoder': ['transformers', 'LlamaModel'], 'text_encoder_2': ['transformers', 'CLIPTextModel'], 'tokenizer': ['transformers', 'LlamaTokenizerFast'], 'tokenizer_2': ['transformers', 'CLIPTokenizer'], 'transformer': ['diffusers', 'HunyuanVideoTransformer3DModel'], 'vae': ['diffusers', 'AutoencoderKLHunyuanVideo']}
+[04-20 03:01:56] Loading required components: ['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'vae', 'transformer', 'scheduler']
+
+[04-20 03:01:56] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:01:58] [RunAI Streamer] Overall time to stream 14.0 GiB of all files to cpu: 2.18s, 6.4 GiB/s
+[04-20 03:02:17] Loaded text_encoder: FSDPLlamaModel (sgl-diffusion version). model size: 13.98 GB, consumed GPU mem: 0.81 GB, avail GPU mem: 73.04 GB
+
+[04-20 03:02:18] Using Torch SDPA backend
+[04-20 03:02:18] [RunAI Streamer] Overall time to stream 234.7 MiB of all files to cpu: 0.03s, 7.6 GiB/s
+[04-20 03:02:18] Loaded text_encoder_2: FSDPCLIPTextModel (sgl-diffusion version). model size: 0.23 GB, consumed GPU mem: 0.76 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:02:20] Loaded tokenizer: LlamaTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:02:20] Loaded tokenizer_2: CLIPTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:02:20] Loading vae from /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773/vae. avail mem: 72.27 GB
+[04-20 03:02:20] VAE unexpected keys: ['encoder.conv_in.conv.bias', 'encoder.conv_in.conv.weight', 'encoder.conv_norm_out.bias', 'encoder.conv_norm_out.weight', 'encoder.conv_out.conv.bias', 'encoder.conv_out.conv.weight', 'encoder.down_blocks.0.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.0.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.0.resnets.0.conv1.conv.bias', 'encoder.down_blocks.0.resnets.0.conv1.conv.weight', 'encoder.down_blocks.0.resnets.0.conv2.conv.bias', 'encoder.down_blocks.0.resnets.0.conv2.conv.weight', 'encoder.down_blocks.0.resnets.0.norm1.bias', 'encoder.down_blocks.0.resnets.0.norm1.weight', 'encoder.down_blocks.0.resnets.0.norm2.bias', 'encoder.down_blocks.0.resnets.0.norm2.weight', 'encoder.down_blocks.0.resnets.1.conv1.conv.bias', 'encoder.down_blocks.0.resnets.1.conv1.conv.weight', 'encoder.down_blocks.0.resnets.1.conv2.conv.bias', 'encoder.down_blocks.0.resnets.1.conv2.conv.weight', 'encoder.down_blocks.0.resnets.1.norm1.bias', 'encoder.down_blocks.0.resnets.1.norm1.weight', 'encoder.down_blocks.0.resnets.1.norm2.bias', 'encoder.down_blocks.0.resnets.1.norm2.weight', 'encoder.down_blocks.1.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.1.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.1.resnets.0.conv1.conv.bias', 'encoder.down_blocks.1.resnets.0.conv1.conv.weight', 'encoder.down_blocks.1.resnets.0.conv2.conv.bias', 'encoder.down_blocks.1.resnets.0.conv2.conv.weight', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.1.resnets.0.norm1.bias', 'encoder.down_blocks.1.resnets.0.norm1.weight', 'encoder.down_blocks.1.resnets.0.norm2.bias', 'encoder.down_blocks.1.resnets.0.norm2.weight', 'encoder.down_blocks.1.resnets.1.conv1.conv.bias', 'encoder.down_blocks.1.resnets.1.conv1.conv.weight', 'encoder.down_blocks.1.resnets.1.conv2.conv.bias', 'encoder.down_blocks.1.resnets.1.conv2.conv.weight', 'encoder.down_blocks.1.resnets.1.norm1.bias', 'encoder.down_blocks.1.resnets.1.norm1.weight', 'encoder.down_blocks.1.resnets.1.norm2.bias', 'encoder.down_blocks.1.resnets.1.norm2.weight', 'encoder.down_blocks.2.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.2.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.2.resnets.0.conv1.conv.bias', 'encoder.down_blocks.2.resnets.0.conv1.conv.weight', 'encoder.down_blocks.2.resnets.0.conv2.conv.bias', 'encoder.down_blocks.2.resnets.0.conv2.conv.weight', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.2.resnets.0.norm1.bias', 'encoder.down_blocks.2.resnets.0.norm1.weight', 'encoder.down_blocks.2.resnets.0.norm2.bias', 'encoder.down_blocks.2.resnets.0.norm2.weight', 'encoder.down_blocks.2.resnets.1.conv1.conv.bias', 'encoder.down_blocks.2.resnets.1.conv1.conv.weight', 'encoder.down_blocks.2.resnets.1.conv2.conv.bias', 'encoder.down_blocks.2.resnets.1.conv2.conv.weight', 'encoder.down_blocks.2.resnets.1.norm1.bias', 'encoder.down_blocks.2.resnets.1.norm1.weight', 'encoder.down_blocks.2.resnets.1.norm2.bias', 'encoder.down_blocks.2.resnets.1.norm2.weight', 'encoder.down_blocks.3.resnets.0.conv1.conv.bias', 'encoder.down_blocks.3.resnets.0.conv1.conv.weight', 'encoder.down_blocks.3.resnets.0.conv2.conv.bias', 'encoder.down_blocks.3.resnets.0.conv2.conv.weight', 'encoder.down_blocks.3.resnets.0.norm1.bias', 'encoder.down_blocks.3.resnets.0.norm1.weight', 'encoder.down_blocks.3.resnets.0.norm2.bias', 'encoder.down_blocks.3.resnets.0.norm2.weight', 'encoder.down_blocks.3.resnets.1.conv1.conv.bias', 'encoder.down_blocks.3.resnets.1.conv1.conv.weight', 'encoder.down_blocks.3.resnets.1.conv2.conv.bias', 'encoder.down_blocks.3.resnets.1.conv2.conv.weight', 'encoder.down_blocks.3.resnets.1.norm1.bias', 'encoder.down_blocks.3.resnets.1.norm1.weight', 'encoder.down_blocks.3.resnets.1.norm2.bias', 'encoder.down_blocks.3.resnets.1.norm2.weight', 'encoder.mid_block.attentions.0.group_norm.bias', 'encoder.mid_block.attentions.0.group_norm.weight', 'encoder.mid_block.attentions.0.to_k.bias', 'encoder.mid_block.attentions.0.to_k.weight', 'encoder.mid_block.attentions.0.to_out.0.bias', 'encoder.mid_block.attentions.0.to_out.0.weight', 'encoder.mid_block.attentions.0.to_q.bias', 'encoder.mid_block.attentions.0.to_q.weight', 'encoder.mid_block.attentions.0.to_v.bias', 'encoder.mid_block.attentions.0.to_v.weight', 'encoder.mid_block.resnets.0.conv1.conv.bias', 'encoder.mid_block.resnets.0.conv1.conv.weight', 'encoder.mid_block.resnets.0.conv2.conv.bias', 'encoder.mid_block.resnets.0.conv2.conv.weight', 'encoder.mid_block.resnets.0.norm1.bias', 'encoder.mid_block.resnets.0.norm1.weight', 'encoder.mid_block.resnets.0.norm2.bias', 'encoder.mid_block.resnets.0.norm2.weight', 'encoder.mid_block.resnets.1.conv1.conv.bias', 'encoder.mid_block.resnets.1.conv1.conv.weight', 'encoder.mid_block.resnets.1.conv2.conv.bias', 'encoder.mid_block.resnets.1.conv2.conv.weight', 'encoder.mid_block.resnets.1.norm1.bias', 'encoder.mid_block.resnets.1.norm1.weight', 'encoder.mid_block.resnets.1.norm2.bias', 'encoder.mid_block.resnets.1.norm2.weight', 'quant_conv.bias', 'quant_conv.weight']
+[04-20 03:02:20] Loaded vae: AutoencoderKLHunyuanVideo (sgl-diffusion version). model size: 0.27 GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:02:20] Loading transformer from /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773/transformer. avail mem: 72.27 GB
+[04-20 03:02:20] Loading HunyuanVideoTransformer3DModel from 6 safetensors file(s) , param_dtype: torch.bfloat16
+[04-20 03:02:20] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:02:20] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:02:21] [RunAI Streamer] Overall time to stream 23.9 GiB of all files to cpu: 1.67s, 14.3 GiB/s
+[04-20 03:02:37] Loaded model with 12.82B parameters
+[04-20 03:02:37] Loaded transformer: HunyuanVideoTransformer3DModel (sgl-diffusion version). model size: 23.88 GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:02:37] Loaded scheduler: FlowMatchEulerDiscreteScheduler (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:02:37] Creating pipeline stages...
+[04-20 03:02:37] Compiling transformer with mode: max-autotune-no-cudagraphs
+[04-20 03:02:37] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:02:37] Pipeline instantiated
+[04-20 03:02:37] Worker 0: Initialized device, model, and distributed environment.
+[04-20 03:02:37] Worker 0: Scheduler loop started.
+[04-20 03:02:37] Unsupported resolution: 848x480, output quality may suffer. Supported resolutions: 960x544, 544x960, 832x624, 624x832, 720x720, 1280x720, 720x1280, 832x1104, 1104x832, 960x960
+[04-20 03:02:37] Adjusting number of frames from 65 to 65 based on model
+[04-20 03:02:37] Processing prompt 1/1: <redacted, len=213>
+[04-20 03:02:37] Processing warmup req... (1/1)
+
  0%|          | 0/1 [00:00<?, ?it/s]/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/functions.py:1692: UserWarning: Dynamo detected a call to a `functools.lru_cache`-wrapped function. Dynamo ignores the cache wrapper and directly traces the wrapped function. Silent incorrectness is only a *potential* risk, not something we have observed. Enable TORCH_LOGS="+dynamo" for a DEBUG stack trace.
+  torch._dynamo.utils.warn_once(msg)
+
+[04-20 03:03:12] Warmup req (1/1) processed in 34.61 seconds
+[04-20 03:03:12] Sampling params:
+                       width: 848
+                      height: 480
+                  num_frames: 65
+                         fps: 13
+                      prompt: <redacted, len=213>
+                  neg_prompt: <redacted, len=392>
+                        seed: 42
+                 infer_steps: 30
+      num_outputs_per_prompt: 1
+              guidance_scale: 1.0
+     embedded_guidance_scale: 6
+                    n_tokens: None
+                  flow_shift: 7
+                  image_path: None
+                 save_output: True
+            output_file_path: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts/hunyuanvideo_bf16_skill_5s.mp4
+        
+[04-20 03:03:12] Running pipeline stages: ['InputValidationStage', 'prompt_encoding_stage_primary', 'TimestepPreparationStage', 'LatentPreparationStage', 'DenoisingStage', 'DecodingStage']
+[04-20 03:03:12] [InputValidationStage] started...
+[04-20 03:03:12] [InputValidationStage] finished in 0.0000 seconds
+[04-20 03:03:12] [TextEncodingStage] started...
+[04-20 03:03:12] [TextEncodingStage] finished in 0.3085 seconds
+[04-20 03:03:12] [TimestepPreparationStage] started...
+[04-20 03:03:12] [TimestepPreparationStage] finished in 0.0004 seconds
+[04-20 03:03:12] [LatentPreparationStage] started...
+[04-20 03:03:12] [LatentPreparationStage] finished in 0.0001 seconds
+[04-20 03:03:12] [DenoisingStage] started...
+
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|▎         | 1/30 [00:00<00:07,  3.82it/s]
  7%|▋         | 2/30 [00:01<00:26,  1.05it/s]
 10%|█         | 3/30 [00:03<00:32,  1.19s/it]
 13%|█▎        | 4/30 [00:04<00:33,  1.29s/it]
 17%|█▋        | 5/30 [00:06<00:33,  1.35s/it]
 20%|██        | 6/30 [00:07<00:33,  1.38s/it]
 23%|██▎       | 7/30 [00:08<00:32,  1.41s/it]
 27%|██▋       | 8/30 [00:10<00:31,  1.42s/it]
 30%|███       | 9/30 [00:11<00:30,  1.43s/it]
 33%|███▎      | 10/30 [00:13<00:28,  1.44s/it]
 37%|███▋      | 11/30 [00:14<00:27,  1.44s/it]
 40%|████      | 12/30 [00:16<00:26,  1.45s/it]
 43%|████▎     | 13/30 [00:17<00:24,  1.45s/it]
 47%|████▋     | 14/30 [00:19<00:23,  1.45s/it]
 50%|█████     | 15/30 [00:20<00:21,  1.45s/it]
 53%|█████▎    | 16/30 [00:22<00:20,  1.46s/it]
 57%|█████▋    | 17/30 [00:23<00:18,  1.46s/it]
 60%|██████    | 18/30 [00:25<00:17,  1.46s/it]
 63%|██████▎   | 19/30 [00:26<00:16,  1.46s/it]
 67%|██████▋   | 20/30 [00:27<00:14,  1.46s/it]
 70%|███████   | 21/30 [00:29<00:13,  1.46s/it]
 73%|███████▎  | 22/30 [00:30<00:11,  1.46s/it]
 77%|███████▋  | 23/30 [00:32<00:10,  1.46s/it]
 80%|████████  | 24/30 [00:33<00:08,  1.46s/it]
 83%|████████▎ | 25/30 [00:35<00:07,  1.46s/it]
 87%|████████▋ | 26/30 [00:36<00:05,  1.46s/it]
 90%|█████████ | 27/30 [00:38<00:04,  1.46s/it]
 93%|█████████▎| 28/30 [00:39<00:02,  1.46s/it]
 97%|█████████▋| 29/30 [00:41<00:01,  1.46s/it]
+[04-20 03:03:54] [DenoisingStage] average time per step: 1.4180 seconds
+[04-20 03:03:54] [DenoisingStage] finished in 42.5421 seconds
+[04-20 03:03:54] [DecodingStage] started...
+[04-20 03:04:11] [DecodingStage] finished in 16.6917 seconds
+[04-20 03:04:12] Output saved to /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts/hunyuanvideo_bf16_skill_5s.mp4
+[04-20 03:04:12] Metrics dumped to: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/benchmark/hunyuanvideo_bf16_skill_5s_perf.json
+[04-20 03:04:12] Pixel data generated successfully in 94.61 seconds
+[04-20 03:04:12] Completed batch processing. Generated 1 outputs in 94.61 seconds
+[04-20 03:04:12] Warmed-up request processed in 59.55 seconds (with warmup excluded)
+[04-20 03:04:12] Metrics dumped to: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/benchmark/hunyuanvideo_bf16_skill_5s_perf.json
+[04-20 03:04:12] Generator was garbage collected without being shut down. Attempting to shut down the local server and client.
+[04-20 03:04:19] Worker 0: Shutdown complete.

validation/h100_skill_5s_20260420/logs/generate_fp8_skill_5s.log

@@ -0,0 +1,112 @@
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:04:47] Warmup enabled, the launch time is expected to be longer than usual
+[04-20 03:04:47] server_args: {"model_path": "/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773", "model_id": null, "backend": "sglang", "attention_backend": null, "attention_backend_config": {}, "cache_dit_config": null, "nccl_port": null, "trust_remote_code": false, "revision": null, "num_gpus": 1, "tp_size": 1, "sp_degree": 1, "ulysses_degree": 1, "ring_degree": 1, "dp_size": 1, "dp_degree": 1, "enable_cfg_parallel": false, "hsdp_replicate_dim": 1, "hsdp_shard_dim": 1, "dist_timeout": 3600, "pipeline_class_name": null, "lora_path": null, "lora_nickname": "default", "lora_scale": 1.0, "lora_weight_name": null, "component_paths": {"transformer": "/data/bbuf/hunyuanvideo_fp8_20260420/sglang_transformer"}, "transformer_weights_path": null, "lora_target_modules": null, "dit_cpu_offload": true, "dit_layerwise_offload": null, "dit_offload_prefetch_size": 0.0, "text_encoder_cpu_offload": true, "image_encoder_cpu_offload": true, "vae_cpu_offload": true, "use_fsdp_inference": false, "pin_cpu_memory": true, "ltx2_two_stage_device_mode": null, "comfyui_mode": false, "enable_torch_compile": true, "warmup": true, "warmup_resolutions": null, "warmup_steps": 1, "disable_autocast": false, "master_port": 30005, "host": "127.0.0.1", "port": 30000, "webui": false, "webui_port": 12312, "scheduler_port": 5618, "strict_ports": false, "output_path": "/data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts", "input_save_path": "inputs/uploads", "prompt_file_path": null, "model_paths": {}, "model_loaded": {"transformer": true, "vae": true, "video_vae": true, "audio_vae": true, "video_dit": true, "audio_dit": true, "dual_tower_bridge": true}, "boundary_ratio": null, "base_gpu_id": 0, "disagg_role": "monolithic", "disagg_timeout": 600, "disagg_dispatch_policy": "round_robin", "disagg_mode": false, "disagg_server_addr": null, "encoder_urls": null, "denoiser_urls": null, "decoder_urls": null, "encoder_tp": null, "denoiser_tp": null, "denoiser_sp": null, "denoiser_ulysses": null, "denoiser_ring": null, "decoder_tp": null, "disagg_transfer_pool_size": 268435456, "disagg_p2p_hostname": "127.0.0.1", "disagg_ib_device": null, "pool_work_endpoint": null, "pool_result_endpoint": null, "log_level": "info", "uvicorn_access_log_exclude_prefixes": []}
+[04-20 03:04:47] Local mode: True
+[04-20 03:04:47] Starting server...
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:04:54] Scheduler bind at endpoint: tcp://127.0.0.1:5618
+[04-20 03:04:54] Initializing distributed environment with world_size=1, device=cuda:0, timeout=3600
+[04-20 03:04:54] Setting distributed timeout to 3600 seconds
+[04-20 03:04:55] No pipeline_class_name specified, using model_index.json
+[04-20 03:04:55] Diffusers version: 0.32.0.dev0
+[04-20 03:04:55] Using pipeline from model_index.json: HunyuanVideoPipeline
+[04-20 03:04:55] Loading pipeline modules...
+[04-20 03:04:55] Model already exists locally and is complete
+[04-20 03:04:55] Model path: /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773
+[04-20 03:04:55] Diffusers version: 0.32.0.dev0
+[04-20 03:04:55] Loading pipeline modules from config: {'_class_name': 'HunyuanVideoPipeline', '_diffusers_version': '0.32.0.dev0', 'scheduler': ['diffusers', 'FlowMatchEulerDiscreteScheduler'], 'text_encoder': ['transformers', 'LlamaModel'], 'text_encoder_2': ['transformers', 'CLIPTextModel'], 'tokenizer': ['transformers', 'LlamaTokenizerFast'], 'tokenizer_2': ['transformers', 'CLIPTokenizer'], 'transformer': ['diffusers', 'HunyuanVideoTransformer3DModel'], 'vae': ['diffusers', 'AutoencoderKLHunyuanVideo']}
+[04-20 03:04:55] Loading required components: ['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'vae', 'transformer', 'scheduler']
+
+[04-20 03:04:55] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:04:57] [RunAI Streamer] Overall time to stream 14.0 GiB of all files to cpu: 2.05s, 6.8 GiB/s
+[04-20 03:05:12] Loaded text_encoder: FSDPLlamaModel (sgl-diffusion version). model size: 13.98 GB, consumed GPU mem: 0.81 GB, avail GPU mem: 73.04 GB
+
+[04-20 03:05:13] Using Torch SDPA backend
+[04-20 03:05:13] [RunAI Streamer] Overall time to stream 234.7 MiB of all files to cpu: 0.03s, 7.7 GiB/s
+[04-20 03:05:14] Loaded text_encoder_2: FSDPCLIPTextModel (sgl-diffusion version). model size: 0.23 GB, consumed GPU mem: 0.76 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:05:15] Loaded tokenizer: LlamaTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:05:15] Loaded tokenizer_2: CLIPTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:05:15] Loading vae from /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773/vae. avail mem: 72.27 GB
+[04-20 03:05:15] VAE unexpected keys: ['encoder.conv_in.conv.bias', 'encoder.conv_in.conv.weight', 'encoder.conv_norm_out.bias', 'encoder.conv_norm_out.weight', 'encoder.conv_out.conv.bias', 'encoder.conv_out.conv.weight', 'encoder.down_blocks.0.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.0.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.0.resnets.0.conv1.conv.bias', 'encoder.down_blocks.0.resnets.0.conv1.conv.weight', 'encoder.down_blocks.0.resnets.0.conv2.conv.bias', 'encoder.down_blocks.0.resnets.0.conv2.conv.weight', 'encoder.down_blocks.0.resnets.0.norm1.bias', 'encoder.down_blocks.0.resnets.0.norm1.weight', 'encoder.down_blocks.0.resnets.0.norm2.bias', 'encoder.down_blocks.0.resnets.0.norm2.weight', 'encoder.down_blocks.0.resnets.1.conv1.conv.bias', 'encoder.down_blocks.0.resnets.1.conv1.conv.weight', 'encoder.down_blocks.0.resnets.1.conv2.conv.bias', 'encoder.down_blocks.0.resnets.1.conv2.conv.weight', 'encoder.down_blocks.0.resnets.1.norm1.bias', 'encoder.down_blocks.0.resnets.1.norm1.weight', 'encoder.down_blocks.0.resnets.1.norm2.bias', 'encoder.down_blocks.0.resnets.1.norm2.weight', 'encoder.down_blocks.1.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.1.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.1.resnets.0.conv1.conv.bias', 'encoder.down_blocks.1.resnets.0.conv1.conv.weight', 'encoder.down_blocks.1.resnets.0.conv2.conv.bias', 'encoder.down_blocks.1.resnets.0.conv2.conv.weight', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.1.resnets.0.norm1.bias', 'encoder.down_blocks.1.resnets.0.norm1.weight', 'encoder.down_blocks.1.resnets.0.norm2.bias', 'encoder.down_blocks.1.resnets.0.norm2.weight', 'encoder.down_blocks.1.resnets.1.conv1.conv.bias', 'encoder.down_blocks.1.resnets.1.conv1.conv.weight', 'encoder.down_blocks.1.resnets.1.conv2.conv.bias', 'encoder.down_blocks.1.resnets.1.conv2.conv.weight', 'encoder.down_blocks.1.resnets.1.norm1.bias', 'encoder.down_blocks.1.resnets.1.norm1.weight', 'encoder.down_blocks.1.resnets.1.norm2.bias', 'encoder.down_blocks.1.resnets.1.norm2.weight', 'encoder.down_blocks.2.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.2.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.2.resnets.0.conv1.conv.bias', 'encoder.down_blocks.2.resnets.0.conv1.conv.weight', 'encoder.down_blocks.2.resnets.0.conv2.conv.bias', 'encoder.down_blocks.2.resnets.0.conv2.conv.weight', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.2.resnets.0.norm1.bias', 'encoder.down_blocks.2.resnets.0.norm1.weight', 'encoder.down_blocks.2.resnets.0.norm2.bias', 'encoder.down_blocks.2.resnets.0.norm2.weight', 'encoder.down_blocks.2.resnets.1.conv1.conv.bias', 'encoder.down_blocks.2.resnets.1.conv1.conv.weight', 'encoder.down_blocks.2.resnets.1.conv2.conv.bias', 'encoder.down_blocks.2.resnets.1.conv2.conv.weight', 'encoder.down_blocks.2.resnets.1.norm1.bias', 'encoder.down_blocks.2.resnets.1.norm1.weight', 'encoder.down_blocks.2.resnets.1.norm2.bias', 'encoder.down_blocks.2.resnets.1.norm2.weight', 'encoder.down_blocks.3.resnets.0.conv1.conv.bias', 'encoder.down_blocks.3.resnets.0.conv1.conv.weight', 'encoder.down_blocks.3.resnets.0.conv2.conv.bias', 'encoder.down_blocks.3.resnets.0.conv2.conv.weight', 'encoder.down_blocks.3.resnets.0.norm1.bias', 'encoder.down_blocks.3.resnets.0.norm1.weight', 'encoder.down_blocks.3.resnets.0.norm2.bias', 'encoder.down_blocks.3.resnets.0.norm2.weight', 'encoder.down_blocks.3.resnets.1.conv1.conv.bias', 'encoder.down_blocks.3.resnets.1.conv1.conv.weight', 'encoder.down_blocks.3.resnets.1.conv2.conv.bias', 'encoder.down_blocks.3.resnets.1.conv2.conv.weight', 'encoder.down_blocks.3.resnets.1.norm1.bias', 'encoder.down_blocks.3.resnets.1.norm1.weight', 'encoder.down_blocks.3.resnets.1.norm2.bias', 'encoder.down_blocks.3.resnets.1.norm2.weight', 'encoder.mid_block.attentions.0.group_norm.bias', 'encoder.mid_block.attentions.0.group_norm.weight', 'encoder.mid_block.attentions.0.to_k.bias', 'encoder.mid_block.attentions.0.to_k.weight', 'encoder.mid_block.attentions.0.to_out.0.bias', 'encoder.mid_block.attentions.0.to_out.0.weight', 'encoder.mid_block.attentions.0.to_q.bias', 'encoder.mid_block.attentions.0.to_q.weight', 'encoder.mid_block.attentions.0.to_v.bias', 'encoder.mid_block.attentions.0.to_v.weight', 'encoder.mid_block.resnets.0.conv1.conv.bias', 'encoder.mid_block.resnets.0.conv1.conv.weight', 'encoder.mid_block.resnets.0.conv2.conv.bias', 'encoder.mid_block.resnets.0.conv2.conv.weight', 'encoder.mid_block.resnets.0.norm1.bias', 'encoder.mid_block.resnets.0.norm1.weight', 'encoder.mid_block.resnets.0.norm2.bias', 'encoder.mid_block.resnets.0.norm2.weight', 'encoder.mid_block.resnets.1.conv1.conv.bias', 'encoder.mid_block.resnets.1.conv1.conv.weight', 'encoder.mid_block.resnets.1.conv2.conv.bias', 'encoder.mid_block.resnets.1.conv2.conv.weight', 'encoder.mid_block.resnets.1.norm1.bias', 'encoder.mid_block.resnets.1.norm1.weight', 'encoder.mid_block.resnets.1.norm2.bias', 'encoder.mid_block.resnets.1.norm2.weight', 'quant_conv.bias', 'quant_conv.weight']
+[04-20 03:05:15] Loaded vae: AutoencoderKLHunyuanVideo (sgl-diffusion version). model size: 0.27 GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:05:15] Loading transformer from /data/bbuf/hunyuanvideo_fp8_20260420/sglang_transformer. avail mem: 72.27 GB
+[04-20 03:05:15] Detected ModelOpt FP8 checkpoint. The format is experimental and subject to change.
+[04-20 03:05:15] ModelOpt FP8 diffusion checkpoints currently keep dit_cpu_offload disabled. Layerwise DiT offload stays enabled because the runtime now preserves the restored FP8 tensor strides.
+[04-20 03:05:15] Loading HunyuanVideoTransformer3DModel from 3 safetensors file(s) , param_dtype: None
+[04-20 03:05:15] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:05:15] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:05:16] [RunAI Streamer] Overall time to stream 15.4 GiB of all files to cpu: 1.17s, 13.1 GiB/s
+[04-20 03:05:19] Loaded model with 12.82B parameters
+[04-20 03:05:19] Loaded transformer: HunyuanVideoTransformer3DModel (sgl-diffusion version). model size: 15.45 GB, consumed GPU mem: 15.57 GB, avail GPU mem: 56.71 GB
+
+[04-20 03:05:19] Loaded scheduler: FlowMatchEulerDiscreteScheduler (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 56.71 GB
+
+[04-20 03:05:19] Creating pipeline stages...
+[04-20 03:05:19] Compiling transformer with mode: max-autotune-no-cudagraphs
+[04-20 03:05:19] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:05:19] Pipeline instantiated
+[04-20 03:05:19] Worker 0: Initialized device, model, and distributed environment.
+[04-20 03:05:19] Worker 0: Scheduler loop started.
+[04-20 03:05:19] Unsupported resolution: 848x480, output quality may suffer. Supported resolutions: 960x544, 544x960, 832x624, 624x832, 720x720, 1280x720, 720x1280, 832x1104, 1104x832, 960x960
+[04-20 03:05:19] Adjusting number of frames from 65 to 65 based on model
+[04-20 03:05:19] Processing prompt 1/1: <redacted, len=213>
+[04-20 03:05:19] Processing warmup req... (1/1)
+
  0%|          | 0/1 [00:00<?, ?it/s]/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/functions.py:1692: UserWarning: Dynamo detected a call to a `functools.lru_cache`-wrapped function. Dynamo ignores the cache wrapper and directly traces the wrapped function. Silent incorrectness is only a *potential* risk, not something we have observed. Enable TORCH_LOGS="+dynamo" for a DEBUG stack trace.
+  torch._dynamo.utils.warn_once(msg)
+
+final peak_memory=330455040
+final peak_memory=330455040
+final peak_memory=330455040
+final peak_memory=1660493400
+final peak_memory=1660493400
+final peak_memory=1660493400
+final peak_memory=0
+final peak_memory=0
+final peak_memory=0
+final peak_memory=1081774900
+final peak_memory=1081774900
+final peak_memory=1081774900
+[04-20 03:05:52] Warmup req (1/1) processed in 33.55 seconds
+[04-20 03:05:52] Sampling params:
+                       width: 848
+                      height: 480
+                  num_frames: 65
+                         fps: 13
+                      prompt: <redacted, len=213>
+                  neg_prompt: <redacted, len=392>
+                        seed: 42
+                 infer_steps: 30
+      num_outputs_per_prompt: 1
+              guidance_scale: 1.0
+     embedded_guidance_scale: 6
+                    n_tokens: None
+                  flow_shift: 7
+                  image_path: None
+                 save_output: True
+            output_file_path: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts/hunyuanvideo_fp8_skill_5s.mp4
+        
+[04-20 03:05:52] Running pipeline stages: ['InputValidationStage', 'prompt_encoding_stage_primary', 'TimestepPreparationStage', 'LatentPreparationStage', 'DenoisingStage', 'DecodingStage']
+[04-20 03:05:52] [InputValidationStage] started...
+[04-20 03:05:52] [InputValidationStage] finished in 0.0000 seconds
+[04-20 03:05:52] [TextEncodingStage] started...
+[04-20 03:05:53] [TextEncodingStage] finished in 0.3064 seconds
+[04-20 03:05:53] [TimestepPreparationStage] started...
+[04-20 03:05:53] [TimestepPreparationStage] finished in 0.0003 seconds
+[04-20 03:05:53] [LatentPreparationStage] started...
+[04-20 03:05:53] [LatentPreparationStage] finished in 0.0001 seconds
+[04-20 03:05:53] [DenoisingStage] started...
+
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|▎         | 1/30 [00:00<00:07,  3.90it/s]
  7%|▋         | 2/30 [00:01<00:24,  1.16it/s]
 10%|█         | 3/30 [00:02<00:28,  1.06s/it]
 13%|█▎        | 4/30 [00:04<00:30,  1.15s/it]
 17%|█▋        | 5/30 [00:05<00:30,  1.21s/it]
 20%|██        | 6/30 [00:06<00:29,  1.24s/it]
 23%|██▎       | 7/30 [00:08<00:28,  1.26s/it]
 27%|██▋       | 8/30 [00:09<00:28,  1.27s/it]
 30%|███       | 9/30 [00:10<00:26,  1.28s/it]
 33%|███▎      | 10/30 [00:11<00:25,  1.29s/it]
 37%|███▋      | 11/30 [00:13<00:24,  1.29s/it]
 40%|████      | 12/30 [00:14<00:23,  1.29s/it]
 43%|████▎     | 13/30 [00:15<00:22,  1.30s/it]
 47%|████▋     | 14/30 [00:17<00:20,  1.30s/it]
 50%|█████     | 15/30 [00:18<00:19,  1.30s/it]
 53%|█████▎    | 16/30 [00:19<00:18,  1.30s/it]
 57%|█████▋    | 17/30 [00:21<00:16,  1.30s/it]
 60%|██████    | 18/30 [00:22<00:15,  1.30s/it]
 63%|██████▎   | 19/30 [00:23<00:14,  1.30s/it]
 67%|██████▋   | 20/30 [00:24<00:13,  1.30s/it]
 70%|███████   | 21/30 [00:26<00:11,  1.30s/it]
 73%|███████▎  | 22/30 [00:27<00:10,  1.30s/it]
 77%|███████▋  | 23/30 [00:28<00:09,  1.30s/it]
 80%|████████  | 24/30 [00:30<00:07,  1.30s/it]
 83%|████████▎ | 25/30 [00:31<00:06,  1.30s/it]
 87%|████████▋ | 26/30 [00:32<00:05,  1.30s/it]
 90%|█████████ | 27/30 [00:34<00:03,  1.30s/it]
 93%|█████████▎| 28/30 [00:35<00:02,  1.30s/it]
 97%|█████████▋| 29/30 [00:36<00:01,  1.30s/it]
+[04-20 03:06:30] [DenoisingStage] average time per step: 1.2659 seconds
+[04-20 03:06:30] [DenoisingStage] finished in 37.9797 seconds
+[04-20 03:06:30] [DecodingStage] started...
+[04-20 03:06:47] [DecodingStage] finished in 16.4582 seconds
+[04-20 03:06:47] Output saved to /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts/hunyuanvideo_fp8_skill_5s.mp4
+[04-20 03:06:47] Metrics dumped to: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/benchmark/hunyuanvideo_fp8_skill_5s_perf.json
+[04-20 03:06:47] Pixel data generated successfully in 88.71 seconds
+[04-20 03:06:47] Completed batch processing. Generated 1 outputs in 88.71 seconds
+[04-20 03:06:47] Warmed-up request processed in 54.75 seconds (with warmup excluded)
+[04-20 03:06:47] Metrics dumped to: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/benchmark/hunyuanvideo_fp8_skill_5s_perf.json
+[04-20 03:06:47] Generator was garbage collected without being shut down. Attempting to shut down the local server and client.
+[04-20 03:06:55] Worker 0: Shutdown complete.

validation/h100_skill_5s_20260420/logs/profile_bf16_skill_5s.log

@@ -0,0 +1,99 @@
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:07:34] Warmup enabled, the launch time is expected to be longer than usual
+[04-20 03:07:34] server_args: {"model_path": "/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773", "model_id": null, "backend": "sglang", "attention_backend": null, "attention_backend_config": {}, "cache_dit_config": null, "nccl_port": null, "trust_remote_code": false, "revision": null, "num_gpus": 1, "tp_size": 1, "sp_degree": 1, "ulysses_degree": 1, "ring_degree": 1, "dp_size": 1, "dp_degree": 1, "enable_cfg_parallel": false, "hsdp_replicate_dim": 1, "hsdp_shard_dim": 1, "dist_timeout": 3600, "pipeline_class_name": null, "lora_path": null, "lora_nickname": "default", "lora_scale": 1.0, "lora_weight_name": null, "component_paths": {}, "transformer_weights_path": null, "lora_target_modules": null, "dit_cpu_offload": true, "dit_layerwise_offload": null, "dit_offload_prefetch_size": 0.0, "text_encoder_cpu_offload": true, "image_encoder_cpu_offload": true, "vae_cpu_offload": true, "use_fsdp_inference": false, "pin_cpu_memory": true, "ltx2_two_stage_device_mode": null, "comfyui_mode": false, "enable_torch_compile": true, "warmup": true, "warmup_resolutions": null, "warmup_steps": 1, "disable_autocast": false, "master_port": 30005, "host": "127.0.0.1", "port": 30000, "webui": false, "webui_port": 12312, "scheduler_port": 5569, "strict_ports": false, "output_path": "outputs/", "input_save_path": "inputs/uploads", "prompt_file_path": null, "model_paths": {}, "model_loaded": {"transformer": true, "vae": true, "video_vae": true, "audio_vae": true, "video_dit": true, "audio_dit": true, "dual_tower_bridge": true}, "boundary_ratio": null, "base_gpu_id": 0, "disagg_role": "monolithic", "disagg_timeout": 600, "disagg_dispatch_policy": "round_robin", "disagg_mode": false, "disagg_server_addr": null, "encoder_urls": null, "denoiser_urls": null, "decoder_urls": null, "encoder_tp": null, "denoiser_tp": null, "denoiser_sp": null, "denoiser_ulysses": null, "denoiser_ring": null, "decoder_tp": null, "disagg_transfer_pool_size": 268435456, "disagg_p2p_hostname": "127.0.0.1", "disagg_ib_device": null, "pool_work_endpoint": null, "pool_result_endpoint": null, "log_level": "info", "uvicorn_access_log_exclude_prefixes": []}
+[04-20 03:07:34] Local mode: True
+[04-20 03:07:34] Starting server...
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:07:41] Scheduler bind at endpoint: tcp://127.0.0.1:5569
+[04-20 03:07:41] Initializing distributed environment with world_size=1, device=cuda:0, timeout=3600
+[04-20 03:07:41] Setting distributed timeout to 3600 seconds
+[04-20 03:07:42] No pipeline_class_name specified, using model_index.json
+[04-20 03:07:42] Diffusers version: 0.32.0.dev0
+[04-20 03:07:42] Using pipeline from model_index.json: HunyuanVideoPipeline
+[04-20 03:07:42] Loading pipeline modules...
+[04-20 03:07:42] Model already exists locally and is complete
+[04-20 03:07:42] Model path: /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773
+[04-20 03:07:42] Diffusers version: 0.32.0.dev0
+[04-20 03:07:42] Loading pipeline modules from config: {'_class_name': 'HunyuanVideoPipeline', '_diffusers_version': '0.32.0.dev0', 'scheduler': ['diffusers', 'FlowMatchEulerDiscreteScheduler'], 'text_encoder': ['transformers', 'LlamaModel'], 'text_encoder_2': ['transformers', 'CLIPTextModel'], 'tokenizer': ['transformers', 'LlamaTokenizerFast'], 'tokenizer_2': ['transformers', 'CLIPTokenizer'], 'transformer': ['diffusers', 'HunyuanVideoTransformer3DModel'], 'vae': ['diffusers', 'AutoencoderKLHunyuanVideo']}
+[04-20 03:07:42] Loading required components: ['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'vae', 'transformer', 'scheduler']
+
+[04-20 03:07:42] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:07:44] [RunAI Streamer] Overall time to stream 14.0 GiB of all files to cpu: 2.13s, 6.6 GiB/s
+[04-20 03:07:59] Loaded text_encoder: FSDPLlamaModel (sgl-diffusion version). model size: 13.98 GB, consumed GPU mem: 0.81 GB, avail GPU mem: 73.04 GB
+
+[04-20 03:08:00] Using Torch SDPA backend
+[04-20 03:08:00] [RunAI Streamer] Overall time to stream 234.7 MiB of all files to cpu: 0.03s, 7.3 GiB/s
+[04-20 03:08:00] Loaded text_encoder_2: FSDPCLIPTextModel (sgl-diffusion version). model size: 0.23 GB, consumed GPU mem: 0.76 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:08:01] Loaded tokenizer: LlamaTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:08:01] Loaded tokenizer_2: CLIPTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:08:01] Loading vae from /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773/vae. avail mem: 72.27 GB
+[04-20 03:08:01] VAE unexpected keys: ['encoder.conv_in.conv.bias', 'encoder.conv_in.conv.weight', 'encoder.conv_norm_out.bias', 'encoder.conv_norm_out.weight', 'encoder.conv_out.conv.bias', 'encoder.conv_out.conv.weight', 'encoder.down_blocks.0.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.0.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.0.resnets.0.conv1.conv.bias', 'encoder.down_blocks.0.resnets.0.conv1.conv.weight', 'encoder.down_blocks.0.resnets.0.conv2.conv.bias', 'encoder.down_blocks.0.resnets.0.conv2.conv.weight', 'encoder.down_blocks.0.resnets.0.norm1.bias', 'encoder.down_blocks.0.resnets.0.norm1.weight', 'encoder.down_blocks.0.resnets.0.norm2.bias', 'encoder.down_blocks.0.resnets.0.norm2.weight', 'encoder.down_blocks.0.resnets.1.conv1.conv.bias', 'encoder.down_blocks.0.resnets.1.conv1.conv.weight', 'encoder.down_blocks.0.resnets.1.conv2.conv.bias', 'encoder.down_blocks.0.resnets.1.conv2.conv.weight', 'encoder.down_blocks.0.resnets.1.norm1.bias', 'encoder.down_blocks.0.resnets.1.norm1.weight', 'encoder.down_blocks.0.resnets.1.norm2.bias', 'encoder.down_blocks.0.resnets.1.norm2.weight', 'encoder.down_blocks.1.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.1.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.1.resnets.0.conv1.conv.bias', 'encoder.down_blocks.1.resnets.0.conv1.conv.weight', 'encoder.down_blocks.1.resnets.0.conv2.conv.bias', 'encoder.down_blocks.1.resnets.0.conv2.conv.weight', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.1.resnets.0.norm1.bias', 'encoder.down_blocks.1.resnets.0.norm1.weight', 'encoder.down_blocks.1.resnets.0.norm2.bias', 'encoder.down_blocks.1.resnets.0.norm2.weight', 'encoder.down_blocks.1.resnets.1.conv1.conv.bias', 'encoder.down_blocks.1.resnets.1.conv1.conv.weight', 'encoder.down_blocks.1.resnets.1.conv2.conv.bias', 'encoder.down_blocks.1.resnets.1.conv2.conv.weight', 'encoder.down_blocks.1.resnets.1.norm1.bias', 'encoder.down_blocks.1.resnets.1.norm1.weight', 'encoder.down_blocks.1.resnets.1.norm2.bias', 'encoder.down_blocks.1.resnets.1.norm2.weight', 'encoder.down_blocks.2.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.2.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.2.resnets.0.conv1.conv.bias', 'encoder.down_blocks.2.resnets.0.conv1.conv.weight', 'encoder.down_blocks.2.resnets.0.conv2.conv.bias', 'encoder.down_blocks.2.resnets.0.conv2.conv.weight', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.2.resnets.0.norm1.bias', 'encoder.down_blocks.2.resnets.0.norm1.weight', 'encoder.down_blocks.2.resnets.0.norm2.bias', 'encoder.down_blocks.2.resnets.0.norm2.weight', 'encoder.down_blocks.2.resnets.1.conv1.conv.bias', 'encoder.down_blocks.2.resnets.1.conv1.conv.weight', 'encoder.down_blocks.2.resnets.1.conv2.conv.bias', 'encoder.down_blocks.2.resnets.1.conv2.conv.weight', 'encoder.down_blocks.2.resnets.1.norm1.bias', 'encoder.down_blocks.2.resnets.1.norm1.weight', 'encoder.down_blocks.2.resnets.1.norm2.bias', 'encoder.down_blocks.2.resnets.1.norm2.weight', 'encoder.down_blocks.3.resnets.0.conv1.conv.bias', 'encoder.down_blocks.3.resnets.0.conv1.conv.weight', 'encoder.down_blocks.3.resnets.0.conv2.conv.bias', 'encoder.down_blocks.3.resnets.0.conv2.conv.weight', 'encoder.down_blocks.3.resnets.0.norm1.bias', 'encoder.down_blocks.3.resnets.0.norm1.weight', 'encoder.down_blocks.3.resnets.0.norm2.bias', 'encoder.down_blocks.3.resnets.0.norm2.weight', 'encoder.down_blocks.3.resnets.1.conv1.conv.bias', 'encoder.down_blocks.3.resnets.1.conv1.conv.weight', 'encoder.down_blocks.3.resnets.1.conv2.conv.bias', 'encoder.down_blocks.3.resnets.1.conv2.conv.weight', 'encoder.down_blocks.3.resnets.1.norm1.bias', 'encoder.down_blocks.3.resnets.1.norm1.weight', 'encoder.down_blocks.3.resnets.1.norm2.bias', 'encoder.down_blocks.3.resnets.1.norm2.weight', 'encoder.mid_block.attentions.0.group_norm.bias', 'encoder.mid_block.attentions.0.group_norm.weight', 'encoder.mid_block.attentions.0.to_k.bias', 'encoder.mid_block.attentions.0.to_k.weight', 'encoder.mid_block.attentions.0.to_out.0.bias', 'encoder.mid_block.attentions.0.to_out.0.weight', 'encoder.mid_block.attentions.0.to_q.bias', 'encoder.mid_block.attentions.0.to_q.weight', 'encoder.mid_block.attentions.0.to_v.bias', 'encoder.mid_block.attentions.0.to_v.weight', 'encoder.mid_block.resnets.0.conv1.conv.bias', 'encoder.mid_block.resnets.0.conv1.conv.weight', 'encoder.mid_block.resnets.0.conv2.conv.bias', 'encoder.mid_block.resnets.0.conv2.conv.weight', 'encoder.mid_block.resnets.0.norm1.bias', 'encoder.mid_block.resnets.0.norm1.weight', 'encoder.mid_block.resnets.0.norm2.bias', 'encoder.mid_block.resnets.0.norm2.weight', 'encoder.mid_block.resnets.1.conv1.conv.bias', 'encoder.mid_block.resnets.1.conv1.conv.weight', 'encoder.mid_block.resnets.1.conv2.conv.bias', 'encoder.mid_block.resnets.1.conv2.conv.weight', 'encoder.mid_block.resnets.1.norm1.bias', 'encoder.mid_block.resnets.1.norm1.weight', 'encoder.mid_block.resnets.1.norm2.bias', 'encoder.mid_block.resnets.1.norm2.weight', 'quant_conv.bias', 'quant_conv.weight']
+[04-20 03:08:01] Loaded vae: AutoencoderKLHunyuanVideo (sgl-diffusion version). model size: 0.27 GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:08:01] Loading transformer from /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773/transformer. avail mem: 72.27 GB
+[04-20 03:08:01] Loading HunyuanVideoTransformer3DModel from 6 safetensors file(s) , param_dtype: torch.bfloat16
+[04-20 03:08:01] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:08:01] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:08:03] [RunAI Streamer] Overall time to stream 23.9 GiB of all files to cpu: 1.71s, 13.9 GiB/s
+[04-20 03:08:21] Loaded model with 12.82B parameters
+[04-20 03:08:21] Loaded transformer: HunyuanVideoTransformer3DModel (sgl-diffusion version). model size: 23.88 GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:08:21] Loaded scheduler: FlowMatchEulerDiscreteScheduler (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:08:21] Creating pipeline stages...
+[04-20 03:08:21] Compiling transformer with mode: max-autotune-no-cudagraphs
+[04-20 03:08:21] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:08:21] Pipeline instantiated
+[04-20 03:08:21] Worker 0: Initialized device, model, and distributed environment.
+[04-20 03:08:21] Worker 0: Scheduler loop started.
+[04-20 03:08:21] Unsupported resolution: 848x480, output quality may suffer. Supported resolutions: 960x544, 544x960, 832x624, 624x832, 720x720, 1280x720, 720x1280, 832x1104, 1104x832, 960x960
+[04-20 03:08:21] Adjusting number of frames from 65 to 65 based on model
+[04-20 03:08:21] Processing prompt 1/1: <redacted, len=213>
+[04-20 03:08:21] Processing warmup req... (1/1)
+
  0%|          | 0/1 [00:00<?, ?it/s]/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/functions.py:1692: UserWarning: Dynamo detected a call to a `functools.lru_cache`-wrapped function. Dynamo ignores the cache wrapper and directly traces the wrapped function. Silent incorrectness is only a *potential* risk, not something we have observed. Enable TORCH_LOGS="+dynamo" for a DEBUG stack trace.
+  torch._dynamo.utils.warn_once(msg)
+
+[04-20 03:08:58] Warmup req (1/1) processed in 37.03 seconds
+[04-20 03:08:58] Sampling params:
+                       width: 848
+                      height: 480
+                  num_frames: 65
+                         fps: 13
+                      prompt: <redacted, len=213>
+                  neg_prompt: <redacted, len=392>
+                        seed: 42
+                 infer_steps: 30
+      num_outputs_per_prompt: 1
+              guidance_scale: 1.0
+     embedded_guidance_scale: 6
+                    n_tokens: None
+                  flow_shift: 7
+                  image_path: None
+                 save_output: False
+            output_file_path: outputs/A_cat_and_a_dog_baking_a_cake_together_in_a_kitchen._The_cat_is_carefully_measuring_flour_while_the_20260420-030821_b0169c22.mp4
+        
+[04-20 03:08:58] Running pipeline stages: ['InputValidationStage', 'prompt_encoding_stage_primary', 'TimestepPreparationStage', 'LatentPreparationStage', 'DenoisingStage', 'DecodingStage']
+[04-20 03:08:58] Profiling request: 445b0a02-cb9c-4793-9b49-1e5a102facb4 for 5 steps...
+[04-20 03:08:58] Starting Profiler...
+[04-20 03:08:58] [InputValidationStage] started...
+[04-20 03:08:58] [InputValidationStage] finished in 0.0014 seconds
+[04-20 03:08:58] [TextEncodingStage] started...
+[04-20 03:08:58] [TextEncodingStage] finished in 0.3087 seconds
+[04-20 03:08:58] [TimestepPreparationStage] started...
+[04-20 03:08:58] [TimestepPreparationStage] finished in 0.0004 seconds
+[04-20 03:08:58] [LatentPreparationStage] started...
+[04-20 03:08:58] [LatentPreparationStage] finished in 0.0001 seconds
+[04-20 03:08:58] [DenoisingStage] started...
+
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|▎         | 1/30 [00:00<00:07,  3.82it/s]
  7%|▋         | 2/30 [00:01<00:26,  1.05it/s]
 10%|█         | 3/30 [00:03<00:32,  1.19s/it]
 13%|█▎        | 4/30 [00:04<00:33,  1.29s/it]
 17%|█▋        | 5/30 [00:06<00:33,  1.35s/it]
 20%|██        | 6/30 [00:07<00:33,  1.38s/it]
 23%|██▎       | 7/30 [00:08<00:32,  1.41s/it]
 27%|██▋       | 8/30 [00:13<00:56,  2.55s/it][04-20 03:09:12] Stopping Profiler...
+[04-20 03:09:32] Saved profiler traces to: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/profiler/bf16/445b0a02-cb9c-4793-9b49-1e5a102facb4-5_steps-global-rank0.trace.json.gz
+
 30%|███       | 9/30 [00:33<02:46,  7.93s/it]
 33%|███▎      | 10/30 [00:35<01:58,  5.93s/it]
 37%|███▋      | 11/30 [00:36<01:26,  4.56s/it]
 40%|████      | 12/30 [00:38<01:04,  3.61s/it]
 43%|████▎     | 13/30 [00:39<00:50,  2.96s/it]
 47%|████▋     | 14/30 [00:40<00:40,  2.50s/it]
 50%|█████     | 15/30 [00:42<00:32,  2.19s/it]
 53%|█████▎    | 16/30 [00:43<00:27,  1.96s/it]
 57%|█████▋    | 17/30 [00:45<00:23,  1.81s/it]
 60%|██████    | 18/30 [00:46<00:20,  1.70s/it]
 63%|██████▎   | 19/30 [00:48<00:17,  1.63s/it]
 67%|██████▋   | 20/30 [00:49<00:15,  1.58s/it]
 70%|███████   | 21/30 [00:51<00:13,  1.54s/it]
 73%|███████▎  | 22/30 [00:52<00:12,  1.52s/it]
 77%|███████▋  | 23/30 [00:54<00:10,  1.50s/it]
 80%|████████  | 24/30 [00:55<00:08,  1.49s/it]
 83%|████████▎ | 25/30 [00:57<00:07,  1.48s/it]
 87%|████████▋ | 26/30 [00:58<00:05,  1.47s/it]
 90%|█████████ | 27/30 [00:59<00:04,  1.47s/it]
 93%|█████████▎| 28/30 [01:01<00:02,  1.47s/it]
 97%|█████████▋| 29/30 [01:02<00:01,  1.46s/it]
+[04-20 03:10:02] [DenoisingStage] average time per step: 2.1438 seconds
+[04-20 03:10:02] [DenoisingStage] finished in 64.3171 seconds
+[04-20 03:10:02] [DecodingStage] started...
+[04-20 03:10:20] [DecodingStage] finished in 17.4636 seconds
+[04-20 03:10:21] Pixel data generated successfully in 119.89 seconds
+[04-20 03:10:21] Completed batch processing. Generated 1 outputs in 119.89 seconds
+[04-20 03:10:21] Warmed-up request processed in 82.14 seconds (with warmup excluded)
+[04-20 03:10:21] Generator was garbage collected without being shut down. Attempting to shut down the local server and client.
+[04-20 03:10:29] Worker 0: Shutdown complete.

validation/h100_skill_5s_20260420/logs/profile_fp8_skill_5s.log

@@ -0,0 +1,101 @@
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:10:57] Warmup enabled, the launch time is expected to be longer than usual
+[04-20 03:10:57] server_args: {"model_path": "/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773", "model_id": null, "backend": "sglang", "attention_backend": null, "attention_backend_config": {}, "cache_dit_config": null, "nccl_port": null, "trust_remote_code": false, "revision": null, "num_gpus": 1, "tp_size": 1, "sp_degree": 1, "ulysses_degree": 1, "ring_degree": 1, "dp_size": 1, "dp_degree": 1, "enable_cfg_parallel": false, "hsdp_replicate_dim": 1, "hsdp_shard_dim": 1, "dist_timeout": 3600, "pipeline_class_name": null, "lora_path": null, "lora_nickname": "default", "lora_scale": 1.0, "lora_weight_name": null, "component_paths": {"transformer": "/data/bbuf/hunyuanvideo_fp8_20260420/sglang_transformer"}, "transformer_weights_path": null, "lora_target_modules": null, "dit_cpu_offload": true, "dit_layerwise_offload": null, "dit_offload_prefetch_size": 0.0, "text_encoder_cpu_offload": true, "image_encoder_cpu_offload": true, "vae_cpu_offload": true, "use_fsdp_inference": false, "pin_cpu_memory": true, "ltx2_two_stage_device_mode": null, "comfyui_mode": false, "enable_torch_compile": true, "warmup": true, "warmup_resolutions": null, "warmup_steps": 1, "disable_autocast": false, "master_port": 30005, "host": "127.0.0.1", "port": 30000, "webui": false, "webui_port": 12312, "scheduler_port": 5573, "strict_ports": false, "output_path": "outputs/", "input_save_path": "inputs/uploads", "prompt_file_path": null, "model_paths": {}, "model_loaded": {"transformer": true, "vae": true, "video_vae": true, "audio_vae": true, "video_dit": true, "audio_dit": true, "dual_tower_bridge": true}, "boundary_ratio": null, "base_gpu_id": 0, "disagg_role": "monolithic", "disagg_timeout": 600, "disagg_dispatch_policy": "round_robin", "disagg_mode": false, "disagg_server_addr": null, "encoder_urls": null, "denoiser_urls": null, "decoder_urls": null, "encoder_tp": null, "denoiser_tp": null, "denoiser_sp": null, "denoiser_ulysses": null, "denoiser_ring": null, "decoder_tp": null, "disagg_transfer_pool_size": 268435456, "disagg_p2p_hostname": "127.0.0.1", "disagg_ib_device": null, "pool_work_endpoint": null, "pool_result_endpoint": null, "log_level": "info", "uvicorn_access_log_exclude_prefixes": []}
+[04-20 03:10:57] Local mode: True
+[04-20 03:10:57] Starting server...
+No platform detected. Using base SRTPlatform with defaults.
+[04-20 03:11:04] Scheduler bind at endpoint: tcp://127.0.0.1:5573
+[04-20 03:11:05] Initializing distributed environment with world_size=1, device=cuda:0, timeout=3600
+[04-20 03:11:05] Setting distributed timeout to 3600 seconds
+[04-20 03:11:05] No pipeline_class_name specified, using model_index.json
+[04-20 03:11:05] Diffusers version: 0.32.0.dev0
+[04-20 03:11:05] Using pipeline from model_index.json: HunyuanVideoPipeline
+[04-20 03:11:05] Loading pipeline modules...
+[04-20 03:11:05] Model already exists locally and is complete
+[04-20 03:11:05] Model path: /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773
+[04-20 03:11:05] Diffusers version: 0.32.0.dev0
+[04-20 03:11:05] Loading pipeline modules from config: {'_class_name': 'HunyuanVideoPipeline', '_diffusers_version': '0.32.0.dev0', 'scheduler': ['diffusers', 'FlowMatchEulerDiscreteScheduler'], 'text_encoder': ['transformers', 'LlamaModel'], 'text_encoder_2': ['transformers', 'CLIPTextModel'], 'tokenizer': ['transformers', 'LlamaTokenizerFast'], 'tokenizer_2': ['transformers', 'CLIPTokenizer'], 'transformer': ['diffusers', 'HunyuanVideoTransformer3DModel'], 'vae': ['diffusers', 'AutoencoderKLHunyuanVideo']}
+[04-20 03:11:05] Loading required components: ['text_encoder', 'text_encoder_2', 'tokenizer', 'tokenizer_2', 'vae', 'transformer', 'scheduler']
+
+[04-20 03:11:05] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:11:08] [RunAI Streamer] Overall time to stream 14.0 GiB of all files to cpu: 2.17s, 6.4 GiB/s
+[04-20 03:11:23] Loaded text_encoder: FSDPLlamaModel (sgl-diffusion version). model size: 13.98 GB, consumed GPU mem: 0.81 GB, avail GPU mem: 73.04 GB
+
+[04-20 03:11:23] Using Torch SDPA backend
+[04-20 03:11:23] [RunAI Streamer] Overall time to stream 234.7 MiB of all files to cpu: 0.03s, 8.1 GiB/s
+[04-20 03:11:24] Loaded text_encoder_2: FSDPCLIPTextModel (sgl-diffusion version). model size: 0.23 GB, consumed GPU mem: 0.76 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:11:25] Loaded tokenizer: LlamaTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+
+[04-20 03:11:25] Loaded tokenizer_2: CLIPTokenizer (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:11:25] Loading vae from /root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773/vae. avail mem: 72.27 GB
+[04-20 03:11:25] VAE unexpected keys: ['encoder.conv_in.conv.bias', 'encoder.conv_in.conv.weight', 'encoder.conv_norm_out.bias', 'encoder.conv_norm_out.weight', 'encoder.conv_out.conv.bias', 'encoder.conv_out.conv.weight', 'encoder.down_blocks.0.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.0.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.0.resnets.0.conv1.conv.bias', 'encoder.down_blocks.0.resnets.0.conv1.conv.weight', 'encoder.down_blocks.0.resnets.0.conv2.conv.bias', 'encoder.down_blocks.0.resnets.0.conv2.conv.weight', 'encoder.down_blocks.0.resnets.0.norm1.bias', 'encoder.down_blocks.0.resnets.0.norm1.weight', 'encoder.down_blocks.0.resnets.0.norm2.bias', 'encoder.down_blocks.0.resnets.0.norm2.weight', 'encoder.down_blocks.0.resnets.1.conv1.conv.bias', 'encoder.down_blocks.0.resnets.1.conv1.conv.weight', 'encoder.down_blocks.0.resnets.1.conv2.conv.bias', 'encoder.down_blocks.0.resnets.1.conv2.conv.weight', 'encoder.down_blocks.0.resnets.1.norm1.bias', 'encoder.down_blocks.0.resnets.1.norm1.weight', 'encoder.down_blocks.0.resnets.1.norm2.bias', 'encoder.down_blocks.0.resnets.1.norm2.weight', 'encoder.down_blocks.1.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.1.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.1.resnets.0.conv1.conv.bias', 'encoder.down_blocks.1.resnets.0.conv1.conv.weight', 'encoder.down_blocks.1.resnets.0.conv2.conv.bias', 'encoder.down_blocks.1.resnets.0.conv2.conv.weight', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.1.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.1.resnets.0.norm1.bias', 'encoder.down_blocks.1.resnets.0.norm1.weight', 'encoder.down_blocks.1.resnets.0.norm2.bias', 'encoder.down_blocks.1.resnets.0.norm2.weight', 'encoder.down_blocks.1.resnets.1.conv1.conv.bias', 'encoder.down_blocks.1.resnets.1.conv1.conv.weight', 'encoder.down_blocks.1.resnets.1.conv2.conv.bias', 'encoder.down_blocks.1.resnets.1.conv2.conv.weight', 'encoder.down_blocks.1.resnets.1.norm1.bias', 'encoder.down_blocks.1.resnets.1.norm1.weight', 'encoder.down_blocks.1.resnets.1.norm2.bias', 'encoder.down_blocks.1.resnets.1.norm2.weight', 'encoder.down_blocks.2.downsamplers.0.conv.conv.bias', 'encoder.down_blocks.2.downsamplers.0.conv.conv.weight', 'encoder.down_blocks.2.resnets.0.conv1.conv.bias', 'encoder.down_blocks.2.resnets.0.conv1.conv.weight', 'encoder.down_blocks.2.resnets.0.conv2.conv.bias', 'encoder.down_blocks.2.resnets.0.conv2.conv.weight', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.bias', 'encoder.down_blocks.2.resnets.0.conv_shortcut.conv.weight', 'encoder.down_blocks.2.resnets.0.norm1.bias', 'encoder.down_blocks.2.resnets.0.norm1.weight', 'encoder.down_blocks.2.resnets.0.norm2.bias', 'encoder.down_blocks.2.resnets.0.norm2.weight', 'encoder.down_blocks.2.resnets.1.conv1.conv.bias', 'encoder.down_blocks.2.resnets.1.conv1.conv.weight', 'encoder.down_blocks.2.resnets.1.conv2.conv.bias', 'encoder.down_blocks.2.resnets.1.conv2.conv.weight', 'encoder.down_blocks.2.resnets.1.norm1.bias', 'encoder.down_blocks.2.resnets.1.norm1.weight', 'encoder.down_blocks.2.resnets.1.norm2.bias', 'encoder.down_blocks.2.resnets.1.norm2.weight', 'encoder.down_blocks.3.resnets.0.conv1.conv.bias', 'encoder.down_blocks.3.resnets.0.conv1.conv.weight', 'encoder.down_blocks.3.resnets.0.conv2.conv.bias', 'encoder.down_blocks.3.resnets.0.conv2.conv.weight', 'encoder.down_blocks.3.resnets.0.norm1.bias', 'encoder.down_blocks.3.resnets.0.norm1.weight', 'encoder.down_blocks.3.resnets.0.norm2.bias', 'encoder.down_blocks.3.resnets.0.norm2.weight', 'encoder.down_blocks.3.resnets.1.conv1.conv.bias', 'encoder.down_blocks.3.resnets.1.conv1.conv.weight', 'encoder.down_blocks.3.resnets.1.conv2.conv.bias', 'encoder.down_blocks.3.resnets.1.conv2.conv.weight', 'encoder.down_blocks.3.resnets.1.norm1.bias', 'encoder.down_blocks.3.resnets.1.norm1.weight', 'encoder.down_blocks.3.resnets.1.norm2.bias', 'encoder.down_blocks.3.resnets.1.norm2.weight', 'encoder.mid_block.attentions.0.group_norm.bias', 'encoder.mid_block.attentions.0.group_norm.weight', 'encoder.mid_block.attentions.0.to_k.bias', 'encoder.mid_block.attentions.0.to_k.weight', 'encoder.mid_block.attentions.0.to_out.0.bias', 'encoder.mid_block.attentions.0.to_out.0.weight', 'encoder.mid_block.attentions.0.to_q.bias', 'encoder.mid_block.attentions.0.to_q.weight', 'encoder.mid_block.attentions.0.to_v.bias', 'encoder.mid_block.attentions.0.to_v.weight', 'encoder.mid_block.resnets.0.conv1.conv.bias', 'encoder.mid_block.resnets.0.conv1.conv.weight', 'encoder.mid_block.resnets.0.conv2.conv.bias', 'encoder.mid_block.resnets.0.conv2.conv.weight', 'encoder.mid_block.resnets.0.norm1.bias', 'encoder.mid_block.resnets.0.norm1.weight', 'encoder.mid_block.resnets.0.norm2.bias', 'encoder.mid_block.resnets.0.norm2.weight', 'encoder.mid_block.resnets.1.conv1.conv.bias', 'encoder.mid_block.resnets.1.conv1.conv.weight', 'encoder.mid_block.resnets.1.conv2.conv.bias', 'encoder.mid_block.resnets.1.conv2.conv.weight', 'encoder.mid_block.resnets.1.norm1.bias', 'encoder.mid_block.resnets.1.norm1.weight', 'encoder.mid_block.resnets.1.norm2.bias', 'encoder.mid_block.resnets.1.norm2.weight', 'quant_conv.bias', 'quant_conv.weight']
+[04-20 03:11:25] Loaded vae: AutoencoderKLHunyuanVideo (sgl-diffusion version). model size: 0.27 GB, consumed GPU mem: 0.00 GB, avail GPU mem: 72.27 GB
+[04-20 03:11:25] Loading transformer from /data/bbuf/hunyuanvideo_fp8_20260420/sglang_transformer. avail mem: 72.27 GB
+[04-20 03:11:25] Detected ModelOpt FP8 checkpoint. The format is experimental and subject to change.
+[04-20 03:11:25] ModelOpt FP8 diffusion checkpoints currently keep dit_cpu_offload disabled. Layerwise DiT offload stays enabled because the runtime now preserves the restored FP8 tensor strides.
+[04-20 03:11:25] Loading HunyuanVideoTransformer3DModel from 3 safetensors file(s) , param_dtype: None
+[04-20 03:11:25] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:11:25] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:11:26] [RunAI Streamer] Overall time to stream 15.4 GiB of all files to cpu: 1.19s, 13.0 GiB/s
+[04-20 03:11:29] Loaded model with 12.82B parameters
+[04-20 03:11:29] Loaded transformer: HunyuanVideoTransformer3DModel (sgl-diffusion version). model size: 15.45 GB, consumed GPU mem: 15.57 GB, avail GPU mem: 56.71 GB
+
+[04-20 03:11:29] Loaded scheduler: FlowMatchEulerDiscreteScheduler (sgl-diffusion version). model size: NA GB, consumed GPU mem: 0.00 GB, avail GPU mem: 56.71 GB
+
+[04-20 03:11:29] Creating pipeline stages...
+[04-20 03:11:29] Compiling transformer with mode: max-autotune-no-cudagraphs
+[04-20 03:11:29] Using FlashAttention (FA3 for hopper, FA4 for blackwell) backend
+[04-20 03:11:29] Pipeline instantiated
+[04-20 03:11:29] Worker 0: Initialized device, model, and distributed environment.
+[04-20 03:11:29] Worker 0: Scheduler loop started.
+[04-20 03:11:29] Unsupported resolution: 848x480, output quality may suffer. Supported resolutions: 960x544, 544x960, 832x624, 624x832, 720x720, 1280x720, 720x1280, 832x1104, 1104x832, 960x960
+[04-20 03:11:29] Adjusting number of frames from 65 to 65 based on model
+[04-20 03:11:29] Processing prompt 1/1: <redacted, len=213>
+[04-20 03:11:29] Processing warmup req... (1/1)
+
  0%|          | 0/1 [00:00<?, ?it/s]/usr/local/lib/python3.12/dist-packages/torch/_dynamo/variables/functions.py:1692: UserWarning: Dynamo detected a call to a `functools.lru_cache`-wrapped function. Dynamo ignores the cache wrapper and directly traces the wrapped function. Silent incorrectness is only a *potential* risk, not something we have observed. Enable TORCH_LOGS="+dynamo" for a DEBUG stack trace.
+  torch._dynamo.utils.warn_once(msg)
+
+[04-20 03:11:59] Warmup req (1/1) processed in 30.53 seconds
+[04-20 03:11:59] Sampling params:
+                       width: 848
+                      height: 480
+                  num_frames: 65
+                         fps: 13
+                      prompt: <redacted, len=213>
+                  neg_prompt: <redacted, len=392>
+                        seed: 42
+                 infer_steps: 30
+      num_outputs_per_prompt: 1
+              guidance_scale: 1.0
+     embedded_guidance_scale: 6
+                    n_tokens: None
+                  flow_shift: 7
+                  image_path: None
+                 save_output: False
+            output_file_path: outputs/A_cat_and_a_dog_baking_a_cake_together_in_a_kitchen._The_cat_is_carefully_measuring_flour_while_the_20260420-031129_103a16c9.mp4
+        
+[04-20 03:11:59] Running pipeline stages: ['InputValidationStage', 'prompt_encoding_stage_primary', 'TimestepPreparationStage', 'LatentPreparationStage', 'DenoisingStage', 'DecodingStage']
+[04-20 03:11:59] Profiling request: 0bb0f929-c3dc-42f5-b834-cfffb815c3e0 for 5 steps...
+[04-20 03:11:59] Starting Profiler...
+[04-20 03:12:00] [InputValidationStage] started...
+[04-20 03:12:00] [InputValidationStage] finished in 0.0019 seconds
+[04-20 03:12:00] [TextEncodingStage] started...
+[04-20 03:12:00] [TextEncodingStage] finished in 0.3088 seconds
+[04-20 03:12:00] [TimestepPreparationStage] started...
+[04-20 03:12:00] [TimestepPreparationStage] finished in 0.0003 seconds
+[04-20 03:12:00] [LatentPreparationStage] started...
+[04-20 03:12:00] [LatentPreparationStage] finished in 0.0001 seconds
+[04-20 03:12:00] [DenoisingStage] started...
+
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|▎         | 1/30 [00:00<00:07,  3.87it/s]
  7%|▋         | 2/30 [00:01<00:24,  1.16it/s]
 10%|█         | 3/30 [00:02<00:28,  1.06s/it]
 13%|█▎        | 4/30 [00:04<00:30,  1.16s/it]
 17%|█▋        | 5/30 [00:05<00:30,  1.21s/it]
 20%|██        | 6/30 [00:06<00:29,  1.24s/it]
 23%|██▎       | 7/30 [00:08<00:29,  1.26s/it]
 27%|██▋       | 8/30 [00:12<00:48,  2.20s/it][04-20 03:12:12] Stopping Profiler...
+[04-20 03:12:28] Saved profiler traces to: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/profiler/fp8/0bb0f929-c3dc-42f5-b834-cfffb815c3e0-5_steps-global-rank0.trace.json.gz
+
 30%|███       | 9/30 [00:28<02:19,  6.63s/it]
 33%|███▎      | 10/30 [00:29<01:39,  4.98s/it]
 37%|███▋      | 11/30 [00:31<01:13,  3.85s/it]
 40%|████      | 12/30 [00:32<00:55,  3.08s/it]
 43%|████▎     | 13/30 [00:33<00:43,  2.54s/it]
 47%|████▋     | 14/30 [00:35<00:34,  2.16s/it]
 50%|█████     | 15/30 [00:36<00:28,  1.90s/it]
 53%|█████▎    | 16/30 [00:37<00:24,  1.72s/it]
 57%|█████▋    | 17/30 [00:39<00:20,  1.59s/it]
 60%|██████    | 18/30 [00:40<00:18,  1.51s/it]
 63%|██████▎   | 19/30 [00:41<00:15,  1.44s/it]
 67%|██████▋   | 20/30 [00:42<00:14,  1.40s/it]
 70%|███████   | 21/30 [00:44<00:12,  1.37s/it]
 73%|███████▎  | 22/30 [00:45<00:10,  1.35s/it]
 77%|███████▋  | 23/30 [00:46<00:09,  1.34s/it]
 80%|████████  | 24/30 [00:48<00:07,  1.33s/it]
 83%|████████▎ | 25/30 [00:49<00:06,  1.32s/it]
 87%|████████▋ | 26/30 [00:50<00:05,  1.32s/it]
 90%|█████████ | 27/30 [00:52<00:03,  1.31s/it]
 93%|█████████▎| 28/30 [00:53<00:02,  1.31s/it]
 97%|█████████▋| 29/30 [00:54<00:01,  1.31s/it]
+[04-20 03:12:56] [DenoisingStage] average time per step: 1.8658 seconds
+[04-20 03:12:56] [DenoisingStage] finished in 55.9762 seconds
+[04-20 03:12:56] [DecodingStage] started...
+[04-20 03:13:13] [DecodingStage] finished in 17.3321 seconds
+[04-20 03:13:14] Pixel data generated successfully in 105.09 seconds
+[04-20 03:13:14] Completed batch processing. Generated 1 outputs in 105.09 seconds
+[04-20 03:13:14] Warmed-up request processed in 73.66 seconds (with warmup excluded)
+[04-20 03:13:14] Generator was garbage collected without being shut down. Attempting to shut down the local server and client.
+[04-20 03:13:22] Worker 0: Shutdown complete.

validation/h100_skill_5s_20260420/profiler/bf16/445b0a02-cb9c-4793-9b49-1e5a102facb4-5_steps-global-rank0.trace.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d6b18f1880bd4a7433e2ed750e6a44f05c689ec88a0c43f79d9dfc02b85d6c7
+size 30644566

validation/h100_skill_5s_20260420/profiler/fp8/0bb0f929-c3dc-42f5-b834-cfffb815c3e0-5_steps-global-rank0.trace.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb1df69a5bf0057fbcf8959df7706ef19e177d088e681192bfe6e3e03c87635
+size 24480956

validation/h100_skill_5s_20260420/profiler/kernel_summary_skill_5s.md

@@ -0,0 +1,73 @@
+# BF16 skill-5s trace
+trace: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/profiler/bf16/445b0a02-cb9c-4793-9b49-1e5a102facb4-5_steps-global-rank0.trace.json.gz
+total CUDA op time: 17055.038 ms
+
+| rank | kernel | total_ms | count | share |
+|---:|---|---:|---:|---:|
+| 1 | `cudaMemcpyAsync` | 7084.313 | 18 | 41.54% |
+| 2 | `void cutlass::device_kernel<flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<2>, cute::C<1>, cute::C<1> >, cute::tuple<cu` | 5456.558 | 409 | 31.99% |
+| 3 | `nvjet_tst_192x208_64x4_2x1_v_bz_coopB_bias_TNT` | 1666.286 | 666 | 9.77% |
+| 4 | `nvjet_tst_192x208_64x4_1x2_h_bz_coopB_bias_TNT` | 1391.623 | 280 | 8.16% |
+| 5 | `nvjet_tst_256x152_64x4_1x2_h_bz_coopA_bias_TNT` | 359.161 | 129 | 2.11% |
+| 6 | `triton_poi_fused_cat_gelu_view_0` | 207.409 | 520 | 1.22% |
+| 7 | `triton_poi_fused_cat_0` | 172.649 | 769 | 1.01% |
+| 8 | `_rotary_embedding_kernel` | 134.757 | 818 | 0.79% |
+| 9 | `_rms_norm_tiled_onepass` | 91.126 | 1076 | 0.53% |
+| 10 | `triton_poi_fused_gelu_0` | 88.732 | 498 | 0.52% |
+| 11 | `fuse_scale_shift_kernel_blc_opt` | 68.166 | 538 | 0.40% |
+| 12 | `kernel_cutlass_kernel_sglangjit_kerneldiffusioncutedslscale_residual_norm_scale_shiftScaleResidualNormScaleShift_object_at__tensorptrbf16_gmem_align32_o_3072div3072div30721___tenso` | 57.168 | 537 | 0.34% |
+| 13 | `triton_poi_fused_clone_select_split_with_sizes_triton_one_pass_rms_norm_cuda_view_0` | 53.547 | 520 | 0.31% |
+| 14 | `triton_poi_fused_clone_select_split_with_sizes_triton_one_pass_rms_norm_cuda_view_1` | 51.039 | 520 | 0.30% |
+| 15 | `cutlass/cutlass_dsl/tvm_ffi_provider.py(576): __call__` | 31.948 | 720 | 0.19% |
+| 16 | `triton_poi_fused_silu_0` | 29.011 | 1137 | 0.17% |
+| 17 | `kernel_cutlass_kernel_sglangjit_kerneldiffusioncutedslscale_residual_norm_scale_shiftScaleResidualNormScaleShift_object_at__tensorptrbf16_gmem_align32_o_3072div3072div30721_tensorp` | 28.657 | 258 | 0.17% |
+| 18 | `triton_poi_fused_clone_select_triton_one_pass_rms_norm_cuda_view_0` | 18.576 | 498 | 0.11% |
+| 19 | `triton_poi_fused_clone_select_triton_one_pass_rms_norm_cuda_view_1` | 17.374 | 498 | 0.10% |
+| 20 | `nvjet_tst_256x8_64x6_4x1_v_bz_bias_TNT` | 10.377 | 256 | 0.06% |
+| 21 | `nvjet_tst_128x8_64x12_4x1_v_bz_bias_TNT` | 7.865 | 280 | 0.05% |
+| 22 | `nvjet_tst_40x64_64x16_4x1_v_bz_bias_TNN` | 6.400 | 288 | 0.04% |
+| 23 | `Memcpy HtoD (Pageable -> Device)` | 5.394 | 12 | 0.03% |
+| 24 | `nvjet_tst_128x56_64x9_4x1_v_bz_bias_TNT` | 4.767 | 141 | 0.03% |
+| 25 | `nvjet_tst_72x64_64x12_2x1_v_bz_bias_TNN` | 3.873 | 141 | 0.02% |
+| 26 | `triton_red_fused__to_copy_add_mul_native_layer_norm_split_unsqueeze_0` | 1.511 | 13 | 0.01% |
+| 27 | `void at::native::vectorized_elementwise_kernel<8, at::native::bfloat16_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda(float)#1}, std::array<char*, 2ul> >(int, at::native::bfloa` | 1.211 | 38 | 0.01% |
+| 28 | `void at::native::(anonymous namespace)::CatArrayBatchedCopy_alignedK_contig<at::native::(anonymous namespace)::OpaqueType<2u>, unsigned int, 3, 128, 1, 8>(at::native::(anonymous na` | 1.205 | 7 | 0.01% |
+| 29 | `triton_poi_fused__to_copy_arange_cat_cos_div_exp_mul_sin_unsqueeze_0` | 0.539 | 36 | 0.00% |
+| 30 | `nvjet_tst_64x8_64x16_4x1_v_bz_bias_TNT` | 0.482 | 43 | 0.00% |
+
+# FP8 skill-5s trace
+trace: /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/profiler/fp8/0bb0f929-c3dc-42f5-b834-cfffb815c3e0-5_steps-global-rank0.trace.json.gz
+total CUDA op time: 15324.257 ms
+
+| rank | kernel | total_ms | count | share |
+|---:|---|---:|---:|---:|
+| 1 | `cudaMemcpyAsync` | 6225.139 | 18 | 40.62% |
+| 2 | `void cutlass::device_kernel<flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<2>, cute::C<1>, cute::C<1> >, cute::tuple<cu` | 5639.818 | 408 | 36.80% |
+| 3 | `_ZN7cutlass13device_kernelINS_4gemm6kernel13GemmUniversalIN4cute5tupleIJiiiiEEENS1_10collective13CollectiveMmaINS1_34MainloopSm90TmaGmmaWarpSpecializedILi6ENS5_IJNS4_1CILi2EEENSA_I` | 1966.159 | 1072 | 12.83% |
+| 4 | `triton_poi_fused_cat_gelu_view_0` | 296.412 | 520 | 1.93% |
+| 5 | `_static_quant_fp8` | 210.237 | 1584 | 1.37% |
+| 6 | `triton_poi_fused_cat_0` | 199.163 | 768 | 1.30% |
+| 7 | `_rotary_embedding_kernel` | 135.674 | 816 | 0.89% |
+| 8 | `triton_poi_fused_clone_select_split_with_sizes_triton_one_pass_rms_norm_cuda_view_1` | 114.999 | 520 | 0.75% |
+| 9 | `_rms_norm_tiled_onepass` | 90.868 | 1072 | 0.59% |
+| 10 | `triton_red_fused__to_copy_add_mul_native_layer_norm_split_unsqueeze_0` | 79.366 | 13 | 0.52% |
+| 11 | `fuse_scale_shift_kernel_blc_opt` | 67.515 | 536 | 0.44% |
+| 12 | `triton_poi_fused_empty_like_gelu_view_0` | 65.944 | 248 | 0.43% |
+| 13 | `kernel_cutlass_kernel_sglangjit_kerneldiffusioncutedslscale_residual_norm_scale_shiftScaleResidualNormScaleShift_object_at__tensorptrbf16_gmem_align32_o_3072div3072div30721___tenso` | 58.068 | 535 | 0.38% |
+| 14 | `triton_poi_fused_clone_select_split_with_sizes_triton_one_pass_rms_norm_cuda_view_0` | 34.725 | 520 | 0.23% |
+| 15 | `kernel_cutlass_kernel_sglangjit_kerneldiffusioncutedslscale_residual_norm_scale_shiftScaleResidualNormScaleShift_object_at__tensorptrbf16_gmem_align32_o_3072div3072div30721_tensorp` | 28.402 | 256 | 0.19% |
+| 16 | `triton_poi_fused_silu_0` | 18.189 | 1135 | 0.12% |
+| 17 | `triton_poi_fused_clone_select_triton_one_pass_rms_norm_cuda_view_0` | 15.576 | 248 | 0.10% |
+| 18 | `triton_poi_fused_clone_select_triton_one_pass_rms_norm_cuda_view_1` | 15.449 | 248 | 0.10% |
+| 19 | `cutlass/cutlass_dsl/tvm_ffi_provider.py(576): __call__` | 14.102 | 720 | 0.09% |
+| 20 | `nvjet_tst_256x8_64x6_4x1_v_bz_bias_TNT` | 10.354 | 254 | 0.07% |
+| 21 | `_ZN7cutlass13device_kernelINS_4gemm6kernel13GemmUniversalIN4cute5tupleIJiiiiEEENS1_10collective13CollectiveMmaINS1_34MainloopSm90TmaGmmaWarpSpecializedILi13ENS5_IJNS4_1CILi1EEENSA_` | 9.062 | 512 | 0.06% |
+| 22 | `nvjet_tst_128x8_64x12_4x1_v_bz_bias_TNT` | 7.892 | 280 | 0.05% |
+| 23 | `Memcpy HtoD (Pageable -> Device)` | 7.046 | 12 | 0.05% |
+| 24 | `triton_poi_fused_empty_like_gelu_view_1` | 2.006 | 248 | 0.01% |
+| 25 | `triton_poi_fused_clone_select_triton_one_pass_rms_norm_cuda_view_2` | 1.926 | 248 | 0.01% |
+| 26 | `triton_poi_fused_clone_select_triton_one_pass_rms_norm_cuda_view_3` | 1.695 | 248 | 0.01% |
+| 27 | `void at::native::(anonymous namespace)::CatArrayBatchedCopy_alignedK_contig<at::native::(anonymous namespace)::OpaqueType<2u>, unsigned int, 3, 128, 1, 8>(at::native::(anonymous na` | 1.236 | 7 | 0.01% |
+| 28 | `void at::native::vectorized_elementwise_kernel<8, at::native::bfloat16_copy_kernel_cuda(at::TensorIteratorBase&)::{lambda(float)#1}, std::array<char*, 2ul> >(int, at::native::bfloa` | 1.213 | 38 | 0.01% |
+| 29 | `triton_poi_fused_clone_permute_view_0` | 0.813 | 25 | 0.01% |
+| 30 | `nvjet_tst_40x64_64x16_4x1_v_bz_bias_TNN` | 0.596 | 30 | 0.00% |

validation/h100_skill_5s_20260420/result_summary_skill_5s.md

@@ -0,0 +1,114 @@
+# HunyuanVideo ModelOpt FP8 H100 skill-5s validation
+
+## Run setup
+- Host/GPU: `h100_sglang`, `CUDA_VISIBLE_DEVICES=0` / rank0 H100.
+- SGLang branch/commit: `codex/hunyuanvideo-modelopt-fp8` at `8f424f4a0`.
+- Native backend gate: logs show `--backend=sglang` and `Using pipeline from model_index.json: HunyuanVideoPipeline`; no diffusers fallback markers were observed.
+- Model: `hunyuanvideo-community/HunyuanVideo` local snapshot `e8c2aaa66fe3742a32c11a6766aecbf07c56e773`.
+- Prompt: `A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window.`
+- Skill preset preserved: `--text-encoder-cpu-offload --pin-cpu-memory --num-frames=65 --width=848 --height=480 --num-inference-steps=30 --save-output --warmup --enable-torch-compile --seed=42`.
+- 5s video choice: added `--fps=13`, producing 65 frames / 13 fps = 5.000s.
+- FP8 run adds `--transformer-path /data/bbuf/hunyuanvideo_fp8_20260420/sglang_transformer`.
+- Note: the same CLI includes `--text-encoder-cpu-offload` and default DiT CPU offload; FP8 logs warn that ModelOpt FP8 keeps `dit_cpu_offload` disabled while preserving layerwise offload behavior.
+
+## Video outputs
+- BF16: `artifacts/hunyuanvideo_bf16_skill_5s.mp4`
+- FP8: `artifacts/hunyuanvideo_fp8_skill_5s.mp4`
+- Contact sheet: `artifacts/hunyuanvideo_bf16_fp8_skill_5s_contact_sheet.png`
+- Probe: `{
+  "BF16": {
+    "width": 848,
+    "height": 480,
+    "r_frame_rate": "13/1",
+    "avg_frame_rate": "13/1",
+    "duration": "5.000000",
+    "nb_frames": "65"
+  },
+  "FP8": {
+    "width": 848,
+    "height": 480,
+    "r_frame_rate": "13/1",
+    "avg_frame_rate": "13/1",
+    "duration": "5.000000",
+    "nb_frames": "65"
+  }
+}`
+
+## Benchmark comparison
+### Performance Comparison Report
+
+#### 1. High-level Summary
+| Metric | Baseline | New | Diff | Status |
+| :--- | :--- | :--- | :--- | :--- |
+| **E2E Latency** | 59545.91 ms | 54747.61 ms | **-4798.30 ms (-8.1%)** | ✅ |
+| **Throughput** | 0.02 req/s | 0.02 req/s | - | - |
+
+
+#### 2. Stage Breakdown
+| Stage Name | Baseline (ms) | New (ms) | Diff (ms) | Diff (%) | Status |
+| :--- | :--- | :--- | :--- | :--- | :--- |
+| InputValidationStage | 0.04 | 0.03 | -0.01 | -17.9% | ⚪️ |
+| TextEncodingStage | 308.48 | 306.44 | -2.04 | -0.7% | ⚪️ |
+| TimestepPreparationStage | 0.41 | 0.34 | -0.07 | -17.3% | ⚪️ |
+| LatentPreparationStage | 0.14 | 0.12 | -0.02 | -14.3% | ⚪️ |
+| DenoisingStage | 42542.10 | 37979.67 | -4562.43 | -10.7% | 🟢 |
+| DecodingStage | 16691.68 | 16458.20 | -233.47 | -1.4% | ⚪️ |
+
+
+<details>
+<summary>Metadata</summary>
+
+- Baseline Commit: `N/A`
+- New Commit: `N/A`
+- Timestamp: 2026-04-20T03:07:09.034563
+</details>
+
+
+## Profiler outputs
+- BF16 trace: `profiler/bf16/445b0a02-cb9c-4793-9b49-1e5a102facb4-5_steps-global-rank0.trace.json.gz`
+- FP8 trace: `profiler/fp8/0bb0f929-c3dc-42f5-b834-cfffb815c3e0-5_steps-global-rank0.trace.json.gz`
+- Kernel summary: `profiler/kernel_summary_skill_5s.md`
+
+### BF16 top CUDA kernels
+| rank | kernel | total_ms | count | share |
+|---:|---|---:|---:|---:|
+| 1 | `cudaMemcpyAsync` | 7084.313 | 18 | 41.54% |
+| 2 | `void cutlass::device_kernel<flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<2>, cute::C<1>, cute::C<1> >, cute::tuple<cu` | 5456.558 | 409 | 31.99% |
+| 3 | `nvjet_tst_192x208_64x4_2x1_v_bz_coopB_bias_TNT` | 1666.286 | 666 | 9.77% |
+| 4 | `nvjet_tst_192x208_64x4_1x2_h_bz_coopB_bias_TNT` | 1391.623 | 280 | 8.16% |
+| 5 | `nvjet_tst_256x152_64x4_1x2_h_bz_coopA_bias_TNT` | 359.161 | 129 | 2.11% |
+| 6 | `triton_poi_fused_cat_gelu_view_0` | 207.409 | 520 | 1.22% |
+| 7 | `triton_poi_fused_cat_0` | 172.649 | 769 | 1.01% |
+| 8 | `_rotary_embedding_kernel` | 134.757 | 818 | 0.79% |
+
+### FP8 top CUDA kernels
+| rank | kernel | total_ms | count | share |
+|---:|---|---:|---:|---:|
+| 1 | `cudaMemcpyAsync` | 6225.139 | 18 | 40.62% |
+| 2 | `void cutlass::device_kernel<flash::enable_sm90_or_later<flash::FlashAttnFwdSm90<flash::CollectiveMainloopFwdSm90<2, cute::tuple<cute::C<2>, cute::C<1>, cute::C<1> >, cute::tuple<cu` | 5639.818 | 408 | 36.80% |
+| 3 | `_ZN7cutlass13device_kernelINS_4gemm6kernel13GemmUniversalIN4cute5tupleIJiiiiEEENS1_10collective13CollectiveMmaINS1_34MainloopSm90TmaGmmaWarpSpecializedILi6ENS5_IJNS4_1CILi2EEENSA_I` | 1966.159 | 1072 | 12.83% |
+| 4 | `triton_poi_fused_cat_gelu_view_0` | 296.412 | 520 | 1.93% |
+| 5 | `_static_quant_fp8` | 210.237 | 1584 | 1.37% |
+| 6 | `triton_poi_fused_cat_0` | 199.163 | 768 | 1.30% |
+| 7 | `_rotary_embedding_kernel` | 135.674 | 816 | 0.89% |
+| 8 | `triton_poi_fused_clone_select_split_with_sizes_triton_one_pass_rms_norm_cuda_view_1` | 114.999 | 520 | 0.75% |
+
+## Commands
+
+### BF16 generation + benchmark
+```bash
+CUDA_VISIBLE_DEVICES=0 PYTHONPATH=python FLASHINFER_DISABLE_VERSION_CHECK=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 sglang generate   --backend=sglang   --model-path=/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773   --prompt='A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window.'   --log-level=info   --seed=42   --text-encoder-cpu-offload   --pin-cpu-memory   --num-frames=65   --fps=13   --width=848   --height=480   --num-inference-steps=30   --save-output   --warmup   --enable-torch-compile   --perf-dump-path /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/benchmark/hunyuanvideo_bf16_skill_5s_perf.json   --output-path /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts   --output-file-name hunyuanvideo_bf16_skill_5s
+```
+
+### FP8 generation + benchmark
+```bash
+CUDA_VISIBLE_DEVICES=0 PYTHONPATH=python FLASHINFER_DISABLE_VERSION_CHECK=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 sglang generate   --backend=sglang   --model-path=/root/.cache/huggingface/hub/models--hunyuanvideo-community--HunyuanVideo/snapshots/e8c2aaa66fe3742a32c11a6766aecbf07c56e773   --transformer-path /data/bbuf/hunyuanvideo_fp8_20260420/sglang_transformer   --prompt='A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window.'   --log-level=info   --seed=42   --text-encoder-cpu-offload   --pin-cpu-memory   --num-frames=65   --fps=13   --width=848   --height=480   --num-inference-steps=30   --save-output   --warmup   --enable-torch-compile   --perf-dump-path /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/benchmark/hunyuanvideo_fp8_skill_5s_perf.json   --output-path /data/bbuf/hunyuanvideo_fp8_20260420_skill5s/artifacts   --output-file-name hunyuanvideo_fp8_skill_5s
+```
+
+### Profiler delta from generation command
+```bash
+export SGLANG_DIFFUSION_TORCH_PROFILER_DIR=/data/bbuf/hunyuanvideo_fp8_20260420_skill5s/profiler/<bf16-or-fp8>
+export SGLANG_TORCH_PROFILER_DIR=$SGLANG_DIFFUSION_TORCH_PROFILER_DIR
+# replace --save-output/--perf-dump-path with:
+--profile --num-profiled-timesteps=5 --no-save-output
+```