Publish validated MiniMax H3 OrbitQuant W4 release

.gitattributes

@@ -0,0 +1,52 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/h264/source-bf16-control.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/h264/w4-text-control.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/h264/w4a4-native-t2va.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/hevc/source-bf16-control.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/hevc/w4-text-control.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/hevc/w4a4-native-t2va.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/reference/astronaut.png filter=lfs diff=lfs merge=lfs -text
+examples/timelines/source-bf16-control.png filter=lfs diff=lfs merge=lfs -text
+examples/timelines/w4-text-control.png filter=lfs diff=lfs merge=lfs -text
+examples/timelines/w4a4-native-t2va.png filter=lfs diff=lfs merge=lfs -text
+comfyui/examples/h264/comfyui-ref2va-608x480.mp4 filter=lfs diff=lfs merge=lfs -text
+comfyui/examples/h264/comfyui-t2va-608x480.mp4 filter=lfs diff=lfs merge=lfs -text
+comfyui/examples/hevc/comfyui-ref2va-608x480.mp4 filter=lfs diff=lfs merge=lfs -text
+comfyui/examples/hevc/comfyui-t2va-608x480.mp4 filter=lfs diff=lfs merge=lfs -text
+comfyui/examples/timelines/comfyui-ref2va-608x480.png filter=lfs diff=lfs merge=lfs -text
+comfyui/examples/timelines/comfyui-t2va-608x480.png filter=lfs diff=lfs merge=lfs -text
+comfyui/workflow-export.png filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,84 @@
+MiniMax H3 COMMUNITY LICENSE AGREEMENT
+MiniMax H3 release date/License date: August 2, 2026.
+The scope of this License Agreement (this “Agreement”) is expressly limited to the “Applicable Territory” as defined below.
+By clicking to accept, or by using, reproducing, modifying, distributing, running, or displaying any portion or element of the MiniMax H3 Works (including through any Hosted Services) in any manner, you acknowledge and accept the terms of this Agreement, and this Agreement shall take immediate effect upon the occurrence of such act.
+I.  Definitions
+1. “Acceptable Use Policy” means the policy published by MiniMax in Exhibit A.
+2. “Agreement” means the terms and conditions set forth herein that govern the use, reproduction, distribution, modification, running, and display of the MiniMax H3 Works or any portion or element thereof.
+3. “Applicable Territory” means worldwide, excluding the Excluded Territories.
+4. “Documentation” means the specifications, manuals, and documentation concerning MiniMax H3 that are publicly released by MiniMax.
+5. “Excluded Territories” means the European Union, the United Kingdom, the Republic of Korea and the United States of America.  
+6. “MiniMax H3” means the video generation model, together with its software and algorithms, including trained model weights, parameters (including optimizer states), machine-learning model code, inference-supporting code, and other elements thereof made publicly available by Us, as released at https://huggingface.co/MiniMaxAI/MiniMax-H3.
+7. “MiniMax H3 Works” means (i) the Materials, (ii) the Model Derivatives, and (iii) all derivatives thereof.
+8. “Hosted Services” means hosted services provided via application programming interfaces (APIs), web access, or any other electronic or remote means.
+9. “Licensee,” “you,” or “your” means the natural or legal person exercising rights and/or using the MiniMax H3 Works for any purpose in any field of use under this Agreement.
+10. “Materials” means, collectively, MiniMax H3 and the Documentation (and any portion thereof), in each case as made available by MiniMax under this Agreement and proprietary to MiniMax.
+11. “Model Derivatives” means all of the following: (i) any modification of MiniMax H3 or any Model Derivative thereof; (ii) any work based on MiniMax H3 or any Model Derivative thereof; or (iii) any other machine learning model created by transferring the patterns of the weights, parameters, operational patterns, or Outputs of MiniMax H3 or any Model Derivative thereof to another model, such that the latter model exhibits behavior similar to MiniMax H3 or its Model Derivatives, including by distillation methods, methods using intermediate data representations, or methods based on training using synthetic-data Outputs generated by MiniMax H3 or its Model Derivatives. For the avoidance of doubt, Outputs are not deemed Model Derivatives.
+12. “Output” means any result of operating or otherwise using MiniMax H3 or any Model Derivatives (including through Hosted Services).
+13. “Third Party” means any natural or legal person that is not under common control with us or with you.
+14. “Including” means “including but not limited to.”
+15. “We,” “Us”  or  “MiniMax” means Nanonoble Pte. Ltd..
+II.  Grant of Rights
+Solely within the Applicable Territory, we grant you a non-exclusive, non-transferable, royalty-free, limited license to use, reproduce, distribute, create derivative works (including Model Derivatives), and modify the Materials in accordance with the terms of this Agreement and the Acceptable Use Policy, based on the intellectual property and other rights owned by MiniMax that are embodied in or used by the Materials. You shall not violate (or encourage or permit any person to violate) any term of this Agreement or the Acceptable Use Policy.
+We will continuously evaluate the applicable laws, regulations and compliance requirements for the Excluded Territories. In the meantime, should any person in such Excluded Territories be interested in deploying our models, you are welcome to contact us about obtaining a license, which will be granted based on robust controls and guardrails for purposes of complying with the laws, regulations and compliance requirements of the Excluded Territories.
+III.  Distribution and Redistribution
+Subject to and conditioned on your continuing compliance with this Agreement, including its territorial restrictions and the Acceptable Use Policy, and solely within the Applicable Territory, you may distribute or make available the MiniMax H3 Works to Third Parties within the Applicable Territory; provided, that all of the following conditions are met:
+1. You must provide a copy of this Agreement to all such Third Parties who receive the MiniMax H3 Works or use your products or services related thereto;
+2. You must cause any modified files to carry prominent notices stating that you have modified such files;
+3. You are encouraged to:
+  a. display a notice on any product or service developed using MiniMax H3 indicating that the product or service is “Powered by MiniMax H3”; 
+  b. add an AI-generation identifier to files produced using generative AI models including MiniMax H3; and
+  c. publish at least one technical blog post or a public statement describing your experience using MiniMax H3 Works;
+4. All distributions to Third Parties (other than through Hosted Services) must be accompanied by a “NOTICE” text file containing the following notice:
+  “MiniMax H3 is licensed under the MiniMax H3 Community License Agreement, Copyright © 2026 MiniMax. All Rights Reserved.”
+You may add your own copyright notices on your modifications; except as provided in this Section and in Section V, however, you may not impose additional or different terms and conditions on the use, reproduction, or distribution of your modifications or of any aggregate Model Derivatives, and your use, reproduction, modification, distribution, running, and display of the work must otherwise comply with the terms and conditions of this Agreement (including the provisions concerning the Applicable Territory). If you receive the MiniMax H3 Works from a Licensee as part of an integrated end-user product, the provisions of Section III of this Agreement do not apply to you, but Section V and Exhibit A remain applicable.
+IV.  Additional Commercial Terms
+1. You shall obtain a separate, prior written authorization from MiniMax by contacting api@minimax.io with the subject line “MiniMax H3 licensing - authorization request”, if your commercial products and services generate more than 20 million US dollars (or equivalent in other currencies) in yearly revenue.
+2. You shall prominently display “MiniMax H3”on the user interface of commercial product or service that uses MiniMax H3 or MiniMax H3 Works.
+V. Use Restrictions
+1. Your use of the MiniMax H3 Works must comply with applicable laws and regulations (including trade-compliance laws and regulations) and must comply with the Acceptable Use Policy for the MiniMax H3 Works, which is incorporated into this Agreement by reference. 
+2. Before providing access to the MiniMax H3 Works or any product, service, or Hosted Service incorporating them, you must bind each recipient or user to enforceable terms at least as protective as the use restrictions in this Section V and Exhibit A, and you must notify each recipient or user that those restrictions apply.
+3. You may not use the MiniMax H3 Works or any of their Outputs or results to improve any other artificial intelligence model (other than MiniMax H3 or its Model Derivatives).
+4. You may not use, reproduce, modify, distribute, or display the MiniMax H3 Works or any of their Outputs or results outside the Applicable Territory. Any such use outside the Applicable Territory is not authorized by this Agreement.
+5. If you provide or make available to any Third Party a product, service, or Hosted Service that permits the generation of Outputs using MiniMax H3 or any Model Derivative, you must, before making that product or service available and throughout its operation, implement, maintain, test, and periodically review reasonable and proportionate technical and organizational safeguards designed to prevent and mitigate access, uses, and Outputs that violate this Section V or Exhibit A, including uses or Outputs that infringe, misappropriate, or otherwise violate any Third Party’s intellectual-property or other rights. You must not knowingly disable, materially weaken, or permit the circumvention of those safeguards. You must maintain a reasonably accessible mechanism for reporting suspected violations. Upon receiving a good-faith report or otherwise obtaining actual knowledge of a violation, you must promptly investigate and take reasonable steps within your control to stop or mitigate the violation, including removing or disabling access to offending content or services and suspending or terminating repeat violators where appropriate. You are responsible for implementing and enforcing these requirements with respect to your products, services, systems, users, and downstream recipients.
+VI.  Intellectual Property
+1. Subject to MiniMax’s rights in the MiniMax H3 Works (and the intellectual property therein), and to your compliance with the terms and conditions of this Agreement, as between you and MiniMax, you will own the derivative works and modifications of the Materials that you have created or had created, as well as any Model Derivatives.
+2. Except for the limited license expressly granted in this paragraph, no trademark license is granted under this Agreement; with respect to MiniMax H3 Works, the Licensee may not use any name or mark owned by or associated with MiniMax or any of its affiliates, except as reasonably and customarily necessary to describe and distribute the MiniMax H3 Works. MiniMax hereby grants you a license to use the “MiniMax H3” mark (the “Mark”) within the Applicable Territory solely for the purpose of complying with Section III.3; provided, that you comply with all applicable trademark-protection laws. All goodwill arising from your use of the Mark shall inure to the benefit of MiniMax.
+3. If you bring or assert any suit or other legal proceeding (including a cross-claim or counterclaim in any action) against us or any other natural or legal person alleging that the Materials, any Output, or any portion of the foregoing infringes any intellectual property right or other right owned by you or for which you can obtain a license, all licenses granted to you under this Agreement will terminate as of the date such suit or proceeding is filed. You shall defend, indemnify, and hold us harmless against any Third-Party claim arising out of or related to the use or distribution of the MiniMax H3 Works by you or by any Third Party.
+4. MiniMax claims no rights over the Outputs you generate. You and your users are entirely responsible for the Outputs and any subsequent use thereof.
+VII.  Disclaimers and Limitations of Liability
+1. We have no obligation to support, update, provide training for, or develop any further version of the MiniMax H3 Works, or to grant any license with respect thereto.
+2. UNLESS AND ONLY TO THE EXTENT REQUIRED BY APPLICABLE LAW, THE MINIMAX H3 WORKS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED “AS IS” WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES OF ANY KIND INCLUDING ANY WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, COURSE OF DEALING, USAGE OF TRADE, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING, REPRODUCING, MODIFYING, PERFORMING, DISPLAYING OR DISTRIBUTING ANY OF THE MINIMAX H3 WORKS OR OUTPUTS AND ASSUME ANY AND ALL RISKS ASSOCIATED WITH YOUR OR A THIRD PARTY’S USE OR DISTRIBUTION OF ANY OF THE MINIMAX H3 WORKS OR OUTPUTS AND YOUR EXERCISE OF RIGHTS AND PERMISSIONS UNDER THIS AGREEMENT.
+3. TO THE FULLEST EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT SHALL MINIMAX OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, FOR ANY DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, EXEMPLARY, CONSEQUENTIAL OR PUNITIVE DAMAGES, OR LOST PROFITS OF ANY KIND ARISING FROM THIS AGREEMENT OR RELATED TO ANY OF THE MINIMAX H3 WORKS OR OUTPUTS, EVEN IF MINIMAX OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING.
+VIII.  Term and Termination
+1. This Agreement is effective from the moment you accept this Agreement or begin accessing the Materials, and, subject to your compliance with its terms and conditions, will remain in effect until terminated as provided herein.
+2. If you breach any term or condition of this Agreement, we have the right to terminate this Agreement. Upon termination, you must immediately cease accessing, using, and distributing the MiniMax H3 Works; delete or destroy all copies within your possession or control; and notify each downstream recipient that your authorization has ended. The obligations in the preceding sentence and Sections VI.1, VI.3, VII, and IX survive termination.
+IX.  Governing Law and Jurisdiction
+1. This Agreement, and any dispute arising out of or related to this Agreement, shall be governed by the laws of the Hong Kong Special Administrative Region of the People’s Republic of China, without regard to its conflict-of-laws rules. The United Nations Convention on Contracts for the International Sale of Goods does not apply to this Agreement.
+2. Any dispute arising out of or related to this Agreement shall be subject to the exclusive jurisdiction of the courts of the Hong Kong Special Administrative Region of the People’s Republic of China with competent jurisdiction. Both MiniMax and the Licensee hereby consent to the exclusive jurisdiction of such courts for any such dispute.
+Additional Note: Please note that the encoder of MiniMax H3 uses Qwen3-VL-32B, which is licensed under Apache 2.0 License: https://github.com/QwenLM/Qwen3-VL/blob/main/LICENSE.
+
+Exhibit A — Acceptable Use Policy
+MiniMax reserves the right to update this Acceptable Use Policy from time to time. 
+Last revised: August 2, 2026.
+MiniMax is committed to promoting the safe and fair use of its tools and features, including MiniMax H3. You agree not to use MiniMax H3, any Model Derivatives, or any Output in any of the following ways:
+1. Use outside the Applicable Territory;
+2. Use in any manner that violates any applicable national, federal, state, local, or international law, regulation, or other legal requirement, or that infringes, misappropriates, or otherwise violates any Third Party’s intellectual-property or other proprietary rights, including through unauthorized reproduction, distribution, public display, public performance, or creation of derivative works;
+3. Use in any manner that may harm yourself or others;
+4. Use to repurpose or distribute the Outputs of MiniMax H3 or any Model Derivatives in order to harm yourself or others;
+5. Use to circumvent or bypass any safety guardrails or safeguards we have implemented;
+6. Use in any manner that exploits or harms, or intends to exploit or harm, minors;
+7. Use to generate or disseminate verifiably false information and/or content for the purpose of harming others or influencing elections;
+8. Use to manufacture or facilitate false online engagement, including fake reviews and other means of false online engagement;
+9. Use to intentionally defame, disparage, or otherwise harass others;
+10. Use to generate and/or disseminate malware (including ransomware) or any other content intended to damage electronic systems;
+11. Use to generate or disseminate personally identifiable information for the purpose of harming others;
+12. Use to generate or disseminate information (including images, code, posts, or articles) in or to any public environment (including via bot tweets or similar means) without clearly and prominently disclosing that such information and/or content is machine-generated;
+13. Use to impersonate another person without that person’s consent, authorization, or lawful right to do so;
+14. Use to make high-risk automated decisions in critical domains that affect individual safety, rights, or well-being (such as law enforcement, immigration, healthcare or medical services, critical-infrastructure management, product-safety components, essential services, credit, employment, housing, education, social scoring, or insurance);
+15. Use in any manner that violates or disregards the social, ethical, or moral standards of other countries or regions;
+16. Use to carry out, assist, threaten, incite, plan, advocate for, or encourage violent extremism or terrorism;
+17. Use for any purpose intended to discriminate against, or harm, individuals or groups based on protected characteristics or categories, online or offline social behavior, or known or predicted personality traits;
+18. Use to intentionally exploit the vulnerabilities of specific populations based on age, social, physical, or psychological characteristics, so as to materially distort the behavior of a member of that group in a manner that causes, or is likely to cause, physical or psychological harm to that person or to others;
+19. Use for military purposes;
+20. Use to engage in any unauthorized or unlicensed professional activity, including but not limited to financial, legal, medical or healthcare, or other professional practice.

MODIFICATIONS.md

@@ -0,0 +1,18 @@
+# Modifications
+
+Eligible learned linear projections in `transformer`, `transformer_ref`, and
+`text_encoder` were converted from the pinned MiniMax H3 checkpoint to
+OrbitQuant W4A4 packed weights. AdaLN projections use OrbitQuant INT4 group-64.
+Embeddings, convolutions, norms, biases, time/context/refiner boundaries, and
+policy-protected input/output projections remain in source precision.
+
+The validated inference runtime loads the packed text encoder and selected
+denoiser with BF16 compute modules so OrbitQuant dispatches every eligible
+linear to the native packed W4A4 Triton path. The text encoder enters CUDA only
+for conditioning, then returns to RAM before the selected denoiser enters CUDA.
+
+The visual and audio VAEs are exact FP32 copies from the pinned source revision;
+they are not quantized and are used only in a separate post-denoise decode
+process. Tokenizer, processor, schedulers, license, and upstream documentation
+are copied without model-weight modification. Validated examples use 50
+scheduler sigma points / 49 denoiser forwards.

NOTICE

@@ -0,0 +1 @@
+MiniMax H3 is licensed under the MiniMax H3 Community License Agreement, Copyright © 2026 MiniMax. All Rights Reserved.

PERFORMANCE.md

@@ -0,0 +1,48 @@
+# Native W4A4 performance correction
+
+Status: validated and released with [OrbitQuant 0.9.1](https://github.com/iamwavecut/OrbitQuant/releases/tag/v0.9.1) at immutable source revision [`73bb2be9add3c86ad60430ec8e2d0a28d7126fc7`](https://github.com/iamwavecut/OrbitQuant/commit/73bb2be9add3c86ad60430ec8e2d0a28d7126fc7).
+
+## Root cause
+
+The prequantized Diffusers loader preserved FP32 when no compute dtype was supplied. H3's non-quantized modules therefore produced FP32 activations, and all 300 OrbitQuant W4 linears rejected the native BF16/FP16 packed kernel. The runtime silently used the generic `triton_cuda` FP32 path instead of `triton_cuda_packed_w4`.
+
+OrbitQuant 0.9.1 changes only the default for prequantized Diffusers artifacts: an omitted dtype becomes BF16, while an explicit dtype remains authoritative. The H3 runner also passes BF16 explicitly for `transformer`, `transformer_ref`, and `text_encoder`. Source `vae` and `audio_vae` loads remain unmodified FP32.
+
+No CUDA/Triton kernel source change was required. The existing packed W4 kernel reached the expected path and timing once activations were BF16. The package also stops independently pinning Triton, leaving PyTorch to provide its matching Triton build.
+
+## Measurements
+
+All GPU measurements used a RunPod NVIDIA RTX PRO 6000 Blackwell Server Edition with PyTorch 2.8.0+cu128 and CUDA 12.8.
+
+| check | result |
+|---|---:|
+| exact full 50-block forward, old FP32 default | 114.1176 s |
+| exact full 50-block forward, explicit BF16 | 2.8687 s |
+| exact full 50-block forward, fixed package default | 2.8668 s |
+| dispatch speedup | 39.8x |
+| fixed packed dispatch | 300 / 300 linears |
+| `transformer_ref` first-process forward with Triton compilation | 5.5383 s |
+| `transformer_ref` hot compiled-cache forward | 2.8904 s |
+| `transformer_ref` packed dispatch | 300 / 300 linears |
+| complete 608x480, 124-frame, 49-forward generation-to-latents stage | 114.6722 s |
+| complete generation peak VRAM | 46,359,644,160 bytes (43.18 GiB) |
+| source-FP32 VAE decode | 4.7095 s |
+| decode peak VRAM | 12,139,359,744 bytes (11.31 GiB) |
+| previous W4A16 compatibility generation-to-latents stage | 2145.7408 s |
+| generation-to-latents speedup vs W4A16 | 18.7x |
+| measured source-BF16 control | 451.5754 s |
+| generation-to-latents speedup vs source BF16 | 3.94x |
+
+The early step-1 native-vs-W4A16 comparison measured 0.0416% video NRMSE and 0.0734% audio NRMSE. The final quality gate passed all 49 model evaluations and 19 reviewed frames. Full-timeline and dense-late inspection found no face breakup, crossed eyes, lip disconnection, ghosting, tile grid, or texture collapse. The final H.265 file is 608x480, 124 readable frames, `hvc1`, with AAC 32 kHz stereo.
+
+## Verification
+
+- OrbitQuant: `uv run ruff check .` passed.
+- OrbitQuant: `uv run pytest -ra` passed with 570 passed and 85 skipped.
+- Linux, Windows, and CPU package CI passed on the integrated release revision.
+- The 0.9.1 wheel was clean-installed from PyPI and reported the expected version.
+- The H3 component-load policy passed on RunPod, including BF16 transformer/text compute and untouched VAE precision.
+- The early native-W4 dtype guard, component-load policy, and media packaging checks passed 14/14 tests on the second RunPod environment.
+- Final HEVC and H.264 fallback independently probe as 608x480, 124 frames, AAC 32 kHz stereo.
+
+The canonical workflow remains two-stage: atomically persist each denoise step and final latents, terminate generation, then decode in a fresh process with source-FP32 VAEs. A direct in-process FP16-latent/FP32-VAE decode is not the validated path.

README.md

@@ -0,0 +1,187 @@
+---
+pipeline_tag: image-text-to-video
+library_name: diffusers
+license: other
+license_name: minimax-h3-community-license-agreement
+license_link: LICENSE
+base_model: MiniMaxAI/MiniMax-H3
+tags:
+  - orbitquant
+  - comfyui
+  - w4
+  - w4a4
+  - native-w4a4-transformer-runtime
+  - text-to-video
+  - image-text-to-video
+  - audio-video-generation
+---
+
+# MiniMax H3 — native OrbitQuant W4A4, source FP32 VAEs
+
+OrbitQuant conversion of [MiniMaxAI/MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3), pinned to source revision [`73372e6cf53e414edd3ab03e357717fb0602e758`](https://huggingface.co/MiniMaxAI/MiniMax-H3/tree/73372e6cf53e414edd3ab03e357717fb0602e758).
+
+Eligible linear weights in `transformer`, `transformer_ref`, and the Qwen3-VL `text_encoder` are stored and executed through OrbitQuant's native packed W4A4 path. BF16 is the compute dtype for non-quantized modules and W4 activations; it is not a full-weight reconstruction. The visual `vae` and `audio_vae` are byte-for-byte FP32 source copies and are never quantized.
+
+## ComfyUI workflow
+
+Download the ready-to-import [MiniMax H3 OrbitQuant T2VA workflow](comfyui/workflows/MiniMax-H3-OrbitQuant-T2VA.json). It is derived from Comfy-Org's bundled [`video_minimax_h3_t2v.json`](https://github.com/Comfy-Org/workflow_templates/blob/7653f1cdef1d92394b6ef9946018c0a8aa4136b8/templates/video_minimax_h3_t2v.json), preserving its human-readable composition: model and usage notes on the left, generation in the center, and the standard ComfyUI `SaveVideo` output on the right.
+
+![MiniMax H3 OrbitQuant public ComfyUI workflow](comfyui/workflow-export.png)
+
+The 6120×2620 PNG above comes from ComfyUI's `Workflow Image → Export → png`
+action rather than a browser screenshot. It embeds the loadable workflow in its
+`tEXtworkflow` metadata.
+
+Install [ComfyUI-OrbitQuant](https://github.com/iamwavecut/ComfyUI-OrbitQuant/tree/feature/minimax-h3-comfyui) in `ComfyUI/custom_nodes`, restart ComfyUI, import the workflow, and set `OrbitQuant Release Loader.model_path` to this downloaded model directory. The tested graph uses the two generic public nodes `OrbitQuant Release Loader` and `OrbitQuant Generate Video`; there are no MiniMax-specific public node classes. Model-specific component selection and offload rules come from [`comfyui_orbitquant.json`](comfyui_orbitquant.json) and an internal allowlisted adapter.
+
+The exact imported graph was executed through ComfyUI `9a9fdb10ed144ce760d9682cb247526ea23cc525` on an RTX PRO 6000 at 608×480, 124 frames, seed 42, and 50 sigma points. Its live-exported [API prompt](comfyui/workflows/MiniMax-H3-OrbitQuant-T2VA-api.json), [machine-readable report](comfyui/report.json), and reviewed T2VA/Ref2VA media are included.
+
+### ComfyUI T2VA — generic public workflow
+
+<video controls playsinline preload="metadata" width="608" aria-label="MiniMax H3 OrbitQuant ComfyUI T2VA proof">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/comfyui/examples/hevc/comfyui-t2va-608x480.mp4" type="video/mp4; codecs=&quot;hvc1&quot;">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/comfyui/examples/h264/comfyui-t2va-608x480.mp4" type="video/mp4">
+  <a href="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/comfyui/examples/h264/comfyui-t2va-608x480.mp4">MiniMax H3 OrbitQuant ComfyUI T2VA proof MP4</a>
+</video>
+
+[H.265/HEVC](comfyui/examples/hevc/comfyui-t2va-608x480.mp4) · [H.264 fallback](comfyui/examples/h264/comfyui-t2va-608x480.mp4) · [five-frame timeline](comfyui/examples/timelines/comfyui-t2va-608x480.png)
+
+### ComfyUI Ref2VA — generic public workflow
+
+<video controls playsinline preload="metadata" width="608" aria-label="MiniMax H3 OrbitQuant ComfyUI Ref2VA proof">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/comfyui/examples/hevc/comfyui-ref2va-608x480.mp4" type="video/mp4; codecs=&quot;hvc1&quot;">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/comfyui/examples/h264/comfyui-ref2va-608x480.mp4" type="video/mp4">
+  <a href="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/comfyui/examples/h264/comfyui-ref2va-608x480.mp4">MiniMax H3 OrbitQuant ComfyUI Ref2VA proof MP4</a>
+</video>
+
+[H.265/HEVC](comfyui/examples/hevc/comfyui-ref2va-608x480.mp4) · [H.264 fallback](comfyui/examples/h264/comfyui-ref2va-608x480.mp4) · [five-frame timeline](comfyui/examples/timelines/comfyui-ref2va-608x480.png)
+
+## Working recipe
+
+1. Move the W4 text encoder onto the GPU only for prompt/reference conditioning.
+2. Offload the text encoder to RAM before denoising.
+3. Move the selected W4 denoiser onto the GPU and run native packed W4A4 with BF16 activations.
+4. Use the official default schedule: **50 sigma-grid points / 49 denoiser forwards**.
+5. Persist every scheduler-step checkpoint atomically.
+6. After denoising exits and releases the GPU, decode the saved latents with the original FP32 visual and audio VAEs.
+
+The earlier eight-point shortcut was the principal quality failure: even the untouched BF16 source produced broken faces, lost temporal detail, and damaged audio with that schedule. VAE loading was separately falsified as the cause by decoding identical bad latents with the exact source VAE.
+
+The apparent native-W4 slowdown had a separate cause: a prequantized Diffusers model loaded without an explicit compute dtype left its non-quantized modules in FP32. All 300 OrbitQuant linears then rejected the BF16/FP16 packed W4 kernel and fell back to the generic FP32 path. OrbitQuant 0.9.1 makes BF16 the default compute dtype for prequantized Diffusers artifacts while preserving an explicitly requested dtype. On the exact 50-block, 11,264-token transformer profile this changed one forward from **114.12 s** to **2.87 s** (39.8×); the package-default load measured **2.87 s** with all 300 linears on `triton_cuda_packed_w4`.
+
+![Source, isolated text component, and final T2VA full-timeline comparisons](examples/quality-timeline-matrix.webp)
+
+Rows from top to bottom: source BF16 control, W4A4 text-only isolation control, final native W4A4 text + transformer. Columns sample frames 0, 31, 62, 92, and 123.
+
+## Embedded examples
+
+### Source BF16 control — dynamic zoom and dialogue
+
+<video controls playsinline preload="metadata" width="608" aria-label="Source BF16 control — dynamic zoom and dialogue">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/hevc/source-bf16-control.mp4" type="video/mp4; codecs=&quot;hvc1&quot;">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/h264/source-bf16-control.mp4" type="video/mp4">
+  <a href="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/h264/source-bf16-control.mp4">Source BF16 control — dynamic zoom and dialogue MP4</a>
+</video>
+
+[H.265/HEVC](examples/hevc/source-bf16-control.mp4) · [H.264 fallback](examples/h264/source-bf16-control.mp4) · [five-frame timeline](examples/timelines/source-bf16-control.png)
+### OrbitQuant W4A4 text encoder, BF16 denoiser
+
+<video controls playsinline preload="metadata" width="608" aria-label="OrbitQuant W4A4 text encoder, BF16 denoiser">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/hevc/w4-text-control.mp4" type="video/mp4; codecs=&quot;hvc1&quot;">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/h264/w4-text-control.mp4" type="video/mp4">
+  <a href="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/h264/w4-text-control.mp4">OrbitQuant W4A4 text encoder, BF16 denoiser MP4</a>
+</video>
+
+[H.265/HEVC](examples/hevc/w4-text-control.mp4) · [H.264 fallback](examples/h264/w4-text-control.mp4) · [five-frame timeline](examples/timelines/w4-text-control.png)
+### Final native W4A4 text + W4A4 T2VA transformer
+
+<video controls playsinline preload="metadata" width="608" aria-label="Final native W4A4 text + W4A4 T2VA transformer">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/hevc/w4a4-native-t2va.mp4" type="video/mp4; codecs=&quot;hvc1&quot;">
+  <source src="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/h264/w4a4-native-t2va.mp4" type="video/mp4">
+  <a href="https://huggingface.co/WaveCut/MiniMax-H3-OrbitQuant-W4A4/resolve/main/examples/h264/w4a4-native-t2va.mp4">Final native W4A4 text + W4A4 T2VA transformer MP4</a>
+</video>
+
+[H.265/HEVC](examples/hevc/w4a4-native-t2va.mp4) · [H.264 fallback](examples/h264/w4a4-native-t2va.mp4) · [five-frame timeline](examples/timelines/w4a4-native-t2va.png)
+
+Each player prefers H.265/HEVC (`hvc1`, CRF 24) and retains H.264 as a browser fallback. Both files carry AAC stereo audio at 32 kHz.
+
+## Component precision and size
+
+| component | stored mode | artifact GiB | eligible linear coverage | OrbitQuant modules | AdaLN INT4 |
+|---|---|---:|---:|---:|---:|
+| `transformer` | `w4a4` | 17.03 | 97.45% | 300 | 50 |
+| `transformer_ref` | `w4a4` | 17.03 | 97.45% | 300 | 50 |
+| `text_encoder` | `w4a4` | 18.55 | 95.80% | 448 | 0 |
+| `vae` | `source_copy` | 9.70 | exact source copy | 0 | 0 |
+| `audio_vae` | `source_copy` | 0.56 | exact source copy | 0 | 0 |
+
+Input/output projections, time/context/refiner boundaries, embeddings, norms, and language-head boundaries that are excluded by the pinned H3/Qwen policy remain in source precision. “Four bit” describes the eligible packed linear weights, not every tensor in the architecture.
+
+## Measurements
+
+Hardware: one RunPod NVIDIA RTX PRO 6000 Blackwell Server Edition (94.97 GiB), PyTorch 2.8.0+cu128. Every sample is exactly **608×480**, **124 frames**, **24 FPS**, and **50 sigma points / 49 forwards**.
+
+| example | task | generation seconds | generation peak VRAM GiB | FP32 VAE decode seconds | decode peak VRAM GiB |
+|---|---|---:|---:|---:|---:|
+| `source-bf16-control` | `t2va` | 451.58 | 63.76 | 19.56 | 11.31 |
+| `w4-text-control` | `t2va` | 381.57 | 63.76 | 13.15 | 11.31 |
+| `w4a4-native-t2va` | `t2va` | 114.67 | 43.18 | 4.71 | 11.31 |
+| `comfyui-t2va` | `t2va` | 137.14 | 18.91 | 5.48 | 11.29 |
+| `comfyui-ref2va` | `ref2va` | 318.86 | 30.46 | 5.50 | 11.29 |
+
+The final native-W4A4 generation through durable latents completed in **114.67 s** at **43.18 GiB** peak VRAM. That is **18.7×** faster than the previously published W4A16 compatibility run (2145.74 s) and **3.94×** faster than the measured source-BF16 control (451.58 s). The first-step native-vs-W4A16 latent drift was 0.0416% video NRMSE and 0.0734% audio NRMSE.
+
+Raw measurements, review decisions, media probes, and early checkpoint comparisons are in [`benchmarks/`](benchmarks/).
+
+## Reproduction
+
+Install the pinned runtimes:
+
+```bash
+pip install "orbitquant[hf,kernels]>=0.9.1,<0.10"
+pip install "diffusers @ git+https://github.com/huggingface/diffusers.git@abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc"
+pip install "transformers>=5.13,<6" accelerate av soundfile
+```
+
+Generate and atomically persist latents:
+
+```bash
+python scripts/run_quantized_example.py \
+  --release . \
+  --output dynamic-zoom.mp4 \
+  --save-latents dynamic-zoom.latents.pt \
+  --prompt "Single continuous live-action shot with fast running, a smooth push-in, stable face identity, Russian dialogue, and rain ambience." \
+  --seed 42 --height 480 --width 608 --num-frames 124 --steps 50 \
+  --manual-stage-offload \
+  --transformer-runtime-mode auto_fused \
+  --checkpoint-dir checkpoints/dynamic-zoom
+```
+
+Only after that process exits, decode with the untouched source VAEs:
+
+```bash
+python scripts/decode_h3_latents.py \
+  --latents dynamic-zoom.latents.pt \
+  --vae vae --vae-dtype fp32 \
+  --audio-vae audio_vae --audio-vae-dtype fp32 \
+  --output dynamic-zoom.mp4
+```
+
+For Ref2VA replace manual stage offload with `--task ref2va --reference examples/reference/astronaut.png --offload-reserve-margin 64GB`. The runner selects `transformer_ref` automatically and lets H3's ComponentsManager place the untouched source FP32 VAE for reference encoding. The verified full ComfyUI generation completed in **318.86 s** at **30.46 GiB** peak VRAM, dispatched 300/300 eligible linears to `triton_cuda_packed_w4`, and returned the text encoder to CPU after conditioning.
+
+## Validation
+
+- The BF16 source control establishes that the official 50-point schedule is temporally coherent.
+- The W4-text-only control was reviewed in full resolution and judged ideal, including audio.
+- The final T2VA example uses native W4A4 conditioning and native W4A4 `transformer` execution before source-FP32 VAE decode.
+- The exact 50-block transformer profile verifies 300/300 eligible linears dispatch to the packed W4 Triton kernel under the package default.
+- The full ComfyUI Ref2VA run completed 49/49 forwards with 300/300 packed-W4 dispatch before source-FP32 visual/audio decode; its reviewed H.264 and HEVC outputs are embedded above.
+- The runner rejects an FP32 native-W4 compute model before the first denoiser forward, preventing a silent generic-kernel fallback from consuming a full run.
+- Every retained review explicitly checks face integrity, motion ghosting, texture breakup, repeating tiles, and at least four full-timeline positions.
+- Every H.264 and HEVC file is independently probed for dimensions, readable frame count, codec, and audio stream.
+- Visual/audio VAE weight-file SHA256 values match the pinned source revision; see [`validation/source_component_copy_audit.json`](validation/source_component_copy_audit.json).
+- Exact artifact hashes are in [`SHA256SUMS`](SHA256SUMS).
+
+## License and modifications
+
+The original [MiniMax H3 Community License Agreement](https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/73372e6cf53e414edd3ab03e357717fb0602e758/LICENSE) is copied as [`LICENSE`](LICENSE). See [`NOTICE`](NOTICE), [`MODIFICATIONS.md`](MODIFICATIONS.md), and the upstream [`QA-about-License`](docs/QA-about-License.md).

SHA256SUMS

@@ -0,0 +1,112 @@
+8116657dfaca81919460b4473c57a69b31a9a6ec97a3b1747df55e59500557e9  .gitattributes
+59b99642b95ea21630e311198ddbfffbfe05aadba0c2f5d884cbdf4efcc90f44  LICENSE
+e6685eda9517bad6751f538dcc0ed967a1a08cacd000a20fe12a3fdd01023b83  MODIFICATIONS.md
+154fe7fbdf198395d53c57d6786b2fbf7e63a083ff70843c5ef78c0c7303d91a  NOTICE
+34fc76ba3797f5d4dbc5e956dde7d1d0e84cfd01c6922094ca1202b05a8bd319  PERFORMANCE.md
+7b140b9f115e729213add98ce65e61c039e11ceb900228d84e109d21275ba046  README.md
+804780f7133477067bd6bbfbc02dc8b3cf9feeb400f97c08f5b1d5f6cbab3840  audio_scheduler/scheduler_config.json
+9a3c645ff892b376c6f5f4c8685964cd75474731af594ff058492a0000caabb6  audio_vae/config.json
+52c59e67ba8de5477c81bfbced0327aabf500f1bfdeefd5ee754529241cb26cb  audio_vae/diffusion_pytorch_model.safetensors
+c088e81b20500cdfcc77da8589f7b1408d080ebabf2051acae8c2f3426ffe1e7  benchmarks/astronaut-w4both-w4a16-early-gate.json
+415a42c8f0c2146ec7cf4bd70c582e5420d87517bef33135cebdd11593d62c9e  benchmarks/astronaut-w4both-w4a16-step001-health.json
+3d3a387b791332d68260f01dcc7c47f2944b21b55a5cbbcc7d966fc095dafa19  benchmarks/early-gate/native-dynamic-zoom-early.metrics.json
+32b73981afb8f5a3deacebf42631b673365ef423f7a6bea2ab41edb42cdf4c73  benchmarks/early-gate/native-vs-w4a16-step001.json
+6422124c7a9fff01207f0f3a0d31c48ee17670def930a27705d98a212c728bc6  benchmarks/early-gate/w4a16-dynamic-zoom-early.metrics.json
+064de721be4f0401f6e852fc4f426375c8d595ab6b30d232fdf270ba0e75747a  benchmarks/example_metrics.csv
+eeff4d8dda3228614ec753d184f908ff9425fd90ca97c6c9e2edcba9b406751d  benchmarks/example_metrics.json
+281bdb716d3e4db82e2e90e74c6bf74f05c483702502ce7df30dd448012dad02  benchmarks/media_probe.json
+47a26334e37b41f5d20068f7131409c0b2fa508ea358a18902e90140837d873c  benchmarks/profiles/native-full-forward-bf16-load.json
+b7a97b2c0d19bcc6bb6e052ec5a552c725083354be375ad12ee6afee38dc4130  benchmarks/profiles/native-full-forward-fp32-load.json
+72c7369bb28fffa42ca6a017c16d63aa86908be2b8f8ff9bc146c4473d394a60  benchmarks/profiles/native-full-forward-package-default.json
+22513428f1923473bae58d43de4b2a9f4bad05e062fa7911b035a6808a49d09e  benchmarks/profiles/native-full-forward-transformer-ref-cold.json
+05e2ed4075c91e75d9834a16dcf0a81397be49639372efe8b2360c73206bb2a9  benchmarks/profiles/native-full-forward-transformer-ref-hot.json
+02233da18b3c01761889df2206181e0fc7b9f120165b37a0fe4c9f7bbdda0096  comfyui/examples/h264/comfyui-ref2va-608x480.mp4
+4cceecedd28dfc593cf5e61145836405d18475bd8109b2ef7a984d96d91a13e6  comfyui/examples/h264/comfyui-t2va-608x480.mp4
+0c3040f81aefdb14b30047a5c972347d012d72dbfa026b8a9b2e245e555b7996  comfyui/examples/hevc/comfyui-ref2va-608x480.mp4
+bc31c456f9fdfae29c4e89d063baa953bbd73141c05a237ed43006336661eaab  comfyui/examples/hevc/comfyui-t2va-608x480.mp4
+78d579a40bf159ed3039900d6503d992843b4ec67662eb777751be908e9237d4  comfyui/examples/timelines/comfyui-ref2va-608x480.png
+7adf6cc9259ae6eb367f8217912bf76ec9cd88ef358150d80b4742d57b50bf01  comfyui/examples/timelines/comfyui-t2va-608x480.png
+f8bbd080f6676741ce5bd4d1c5dfda874d7077a3c3cc566a6e6c2e5fd0ea1a32  comfyui/report.json
+df49d477b234e71c23e0a7d457d66315408333db62433885db4b20f3606328f0  comfyui/workflow-export.png
+295ac493e659067aa1b39b9bf70143922c4459d1897636d896327011f167e1c1  comfyui/workflows/MiniMax-H3-OrbitQuant-Ref2VA-api.json
+5ed9b7600b6869bd887459d4d3e8f740201f6b4bd8ca40d8fd786bb6f8721f29  comfyui/workflows/MiniMax-H3-OrbitQuant-T2VA-api.json
+7c1a82427b80203e4252726a0ab3a336a6859fe2ff43302a07de00bce5722acb  comfyui/workflows/MiniMax-H3-OrbitQuant-T2VA.json
+b248e45ce21157a41c298a491f95ea2bc25468c3fe3cb7eaf939818056853250  comfyui_orbitquant.json
+26900a2868636f886d241efe94002ad11d858b3c3eb005e58c0c2ad60f0de7ae  docs/QA-about-License.md
+87044e59a72a2b8ed3df285de044a097d7366f3004dcfc26ffd11fc71ed32d17  examples/h264/source-bf16-control.mp4
+513f09b20cb4e59a47a2f4ff19e544cb29d41870ee2c4759ad0a2ff6cb16fe0c  examples/h264/w4-text-control.mp4
+679c0b690fce47ed2dd19ba7fa185d0cea69ee3a3b308a84faa909ce0762d24b  examples/h264/w4a4-native-t2va.mp4
+eb4eb5d19d231887c6dfba5be70ac3d9107fe076a5f7c69e6fa77ecc0ad5ced9  examples/hevc/source-bf16-control.mp4
+ea89a3cc2a37c4294da4142f22d9d3778f1d086efb6125d73e1d5ccf392c6cbb  examples/hevc/w4-text-control.mp4
+0bef87dbecbd9963e6febb43fb44cc2caf142192e59f7deee13950e55789b235  examples/hevc/w4a4-native-t2va.mp4
+cc97132c8132fa431d375c1e0b0dc8a85b536a91bc13bd5494e789e592126426  examples/quality-timeline-matrix.webp
+b1257de01d8440baea06865ea35ce557d885b6dda700281ba25202193c58285c  examples/reference/astronaut.png
+1c33ff3ec4f637b15496b01130cd83c7393f742fce45b5986d158e6970cfdd27  examples/reviews/source-bf16-control.json
+fad45c95f068f9abfc5125e1e0993004fe036b793803eebc3e28255827200a25  examples/reviews/w4-text-control.json
+9740fde8102c96aaa2bb9f41ecfbf3eaea651d2393944a21107889789d21f6e7  examples/reviews/w4a4-native-t2va.json
+efa068559ff6c2b7d0f6fbf2c835ba4d1b654f4cd016e7be737e27b50c0c380e  examples/timelines/source-bf16-control.png
+367855aa5fd90e4a6b6baece0894a3f4343a6a6ffc6b16750711d14a05bb82fe  examples/timelines/w4-text-control.png
+a69f89026fde4cacf6a52047ccdf5909a9ba218658d13b9b514a9fd11ece4482  examples/timelines/w4a4-native-t2va.png
+a6c5f6fca6c2c92726ddb03407ff11d9c1287bbfa164e489e58fb84376073faa  modular_model_index.json
+5c72a170d2a4a1a3bc5adad2e689ae28138a9700e5b8c96c0266331e86c0acce  processor/chat_template.json
+599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3  processor/merges.txt
+27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516  processor/preprocessor_config.json
+a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7  processor/tokenizer.json
+a07e942ac874baa13758de8d1fbdb186683cc03416b5589e1b6671c6b3057c68  processor/tokenizer_config.json
+7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13  processor/video_preprocessor_config.json
+ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910  processor/vocab.json
+432518f56fa7e8fc1b6f48f98d1a93606eb1d3ba3aabb028343bcb7f960a1b2a  prompts.json
+95676df192c32b659a46cc8ad1c5b1c7556d8258730eea761399c60abdb94894  quantization_manifest.json
+894c94241bf053846755f74c639e86cceb03335c40bf36745d0c8a3780c61821  runtime-requirements.txt
+8fa6c3aa70dc9e691e1a6df899fd1b6f75f70481a27cee6e18a303817075c304  scheduler/scheduler_config.json
+075a3208f5fba960a4eae3003a5b7fcc57ad3d6a0ba9e8d91391f0d7b64cae04  scripts/audit_source_component_copies.py
+5f970b7f378349add8381ad5fa809d711ef315966d0c3a6f4aa36ae7b72665f9  scripts/checkpoint_io.py
+b1bb1643636c0a83b31eed65996ab4a8c8bc9c2f458bae5a5dccde02158ce432  scripts/decode_h3_latents.py
+5f58a74b4408abf18e0617dd35a999ce0f8b8d238fd69f45e163ae0171421145  scripts/decode_precision_policy.py
+e9ea546b28f1fc4ec01f1270072280be684d6d92cea3546e268b9b5d3e5b1c2d  scripts/latent_io.py
+ea79fe8d056bbfe08e5f9ddfadd83f74d11539641221679e07a2a82dc2cf5086  scripts/manual_stage_offload.py
+5d1fb5bc4f429447d68f7647a566e90320d55f7ac6b630a320be39642e1f202f  scripts/media_packaging.py
+655098f3ec10bdbb41bb79b2d38a326d5d7aad115beec90edbfd46611446676a  scripts/offload_policy.py
+64b2b294d46ce22b3fe9a3e7642e3b1344a190f4a83dd9bb44a3a48204b541d5  scripts/orbitquant_h3_compat.py
+8eb302a23053b6d96ecea86974cb55cc5193eb295fc41c314a34c452779f9a07  scripts/quality_gate.py
+295f3c21780891df90f9876c822fb46425dffefbfd05e16e370585ed04af62d9  scripts/run_quantized_example.py
+841b69c59e4eff19737414b9547599c138581dded744467ab8c6b86d9005a009  scripts/runtime_cache_policy.py
+496abd4859f92b354ae9b34d2028864ea1a3c13d92f7067164aea5465d4ba61d  scripts/verification_policy.py
+de100ce1d13e83fe378b1a8543b2e906ce2cecc4e7153d126864975c5378a7ed  scripts/verify_components.py
+a2b186303b94eeb97750e4cc498103217c1888794e160c7c6b79dd39220267a9  text_encoder/config.json
+19c0e38d660f3366f146fba332780419eab679529fb6381783b1cacc7b863d53  text_encoder/generation_config.json
+7461c06c1ee7f5e606bd014e1825183407fcbc18893ef3ec7099398b8274a20f  text_encoder/model-00001-of-00005.safetensors
+c510d3757606e1cf375e983bf54ff1c2c0ba470c83b74679224bcf8d51745a5f  text_encoder/model-00002-of-00005.safetensors
+988e2362f4fbe353b761afddff94c69a64eb821409fe226fe835d252edb056ec  text_encoder/model-00003-of-00005.safetensors
+bfb4cb2a0c7e64ee2960596a5049a797e6a45776c96913e06ee8c1964d11125d  text_encoder/model-00004-of-00005.safetensors
+3cf949a827326855d2dc847557c2f71bb0d464c9a9cafb444b6c2e0aac86b742  text_encoder/model-00005-of-00005.safetensors
+36b2d7912dffc98f6f5e0bd7b11234c451f6c99f351f5f28a87ce30979f73271  text_encoder/model.safetensors.index.json
+599bab54075088774b1733fde865d5bd747cbcc7a547c5bc12610e874e26f5e3  tokenizer/merges.txt
+a5d85b6dcc535e6b93115a9ef287e6132fdbf30270da6218194ba742261173c7  tokenizer/tokenizer.json
+a07e942ac874baa13758de8d1fbdb186683cc03416b5589e1b6671c6b3057c68  tokenizer/tokenizer_config.json
+ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910  tokenizer/vocab.json
+eac2b97122aeab6cb7a324fd8b451e4ad021777d6137d285e2098b089bafb926  transformer/config.json
+fa30399f163a85ab89d7e23a93237d8188ea1bf09a6bbd489e0f82102796daa7  transformer/diffusion_pytorch_model-00001-of-00005.safetensors
+6d929ad9edbedd0eacde8ecac20f227ce543b1f63edaf7b283340a485e319bdc  transformer/diffusion_pytorch_model-00002-of-00005.safetensors
+65a6880aad54e1dac2aa64bbe29fbf1199f179a5f6d5b16dd0e9e3448322811f  transformer/diffusion_pytorch_model-00003-of-00005.safetensors
+8cc4dd8cd6e5dbe8d7610a361ef6fcbbfd7a2a0e4a61fffc73b116f88aad13a7  transformer/diffusion_pytorch_model-00004-of-00005.safetensors
+def316191099ee999ee21c7a34b6954c74ffe18d4b7faf92847457afab494cc9  transformer/diffusion_pytorch_model-00005-of-00005.safetensors
+127932208cc61680a73ed835cdbf971a8190c7c275668864a855a15c614cb765  transformer/diffusion_pytorch_model.safetensors.index.json
+eac2b97122aeab6cb7a324fd8b451e4ad021777d6137d285e2098b089bafb926  transformer_ref/config.json
+357baa234bc04ed4843eefe19ce98a23995e24f012bcfc5117b464c9fc649815  transformer_ref/diffusion_pytorch_model-00001-of-00005.safetensors
+60b0663b551b9cf1ccc62c14571689c3cf54de8efef0971b1687cbbed9ba2149  transformer_ref/diffusion_pytorch_model-00002-of-00005.safetensors
+ee2c6a0690df2c2d13e21c82cd6c25ef499a5e317d71944ce70cb83f1747b1b0  transformer_ref/diffusion_pytorch_model-00003-of-00005.safetensors
+a353924509eff7dd7be99f3efc8c57eecf4d87e24976ab2b02e2367141891651  transformer_ref/diffusion_pytorch_model-00004-of-00005.safetensors
+006e19769149a3f3d96a5df7341bca346cfb0113d6e2b1b984167acd3daf5e12  transformer_ref/diffusion_pytorch_model-00005-of-00005.safetensors
+127932208cc61680a73ed835cdbf971a8190c7c275668864a855a15c614cb765  transformer_ref/diffusion_pytorch_model.safetensors.index.json
+78f67deec3d63aae807f2bfe7154bc1e26f6372cb20b63265fcbae1b62bb5745  vae/config.json
+72f4c6be84ac0674f27398cde991dd9d719762f3952c4921aa66b2ce542f6374  vae/diffusion_pytorch_model-00001-of-00003.safetensors
+2e05e8bc23fa4071043e17fd242be8acd0685e781a43987432b2eae925be4198  vae/diffusion_pytorch_model-00002-of-00003.safetensors
+c05d6ac4b1a33de372799d708531da6320f6a3ce6d1ce6d895e770988e004a39  vae/diffusion_pytorch_model-00003-of-00003.safetensors
+15f6d44553c3c616b0dc999920aa784f92ecee7e4201f1f99ac405cfbf3061ca  vae/diffusion_pytorch_model.safetensors.index.json
+d79d95cb98ff188056ff93668935f434cde9374f64ac7f683141d868e23a7ffe  validation/audio_vae.json
+bb84e8c4b30042de4306035ac8c7b401bddc6fcadb33336bd9933993d9c8bb3a  validation/source_component_copy_audit.json
+43e8558e13737d94de0e5739adf10c4e371b00149d5c5965eb8557042749a6fc  validation/text_encoder.json
+34b7c710e3a1a13ae078f497064703f898b10d782a67d3861efb29c7a0a14c02  validation/transformer.json
+8ac5758c586d3b2ac170e6766f0fc5c191ad6e3e7d8de2a372961df29308eda6  validation/transformer_ref.json
+18e908a81caf2a84520fd1813b40948867bcc9d1a74f6029f800779cf1417a77  validation/vae.json

audio_scheduler/scheduler_config.json

@@ -0,0 +1,5 @@
+{
+  "_class_name": "MiniMaxH3Scheduler",
+  "_diffusers_version": "0.36.0.dev0",
+  "shift": 3.0
+}

audio_vae/config.json

@@ -0,0 +1,125 @@
+{
+  "_class_name": "AutoencoderKLMiniMaxH3Audio",
+  "_diffusers_version": "0.36.0.dev0",
+  "encoder_dim": 64,
+  "encoder_rates": [
+    2,
+    4,
+    4,
+    5,
+    5
+  ],
+  "latent_dim": 2048,
+  "latent_channels": 32,
+  "decoder_dim": 1024,
+  "decoder_rates": [
+    5,
+    5,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "decoder_kernel_sizes": [
+    9,
+    9,
+    4,
+    4,
+    4,
+    4,
+    4
+  ],
+  "num_attention_heads": 8,
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "sampling_rate": 32000,
+  "latents_mean": [
+    -0.020211687488382354,
+    0.3876466479950502,
+    -0.04398279799186767,
+    -0.28591514936373,
+    0.08179686214561671,
+    -0.35782641352446604,
+    0.040623809960919084,
+    -0.01552534501956604,
+    -0.223362481667332,
+    0.1821006842509091,
+    0.2941778783780663,
+    -0.07901167601970885,
+    -0.056815072777201,
+    -0.3699028221860095,
+    -0.31616315591624855,
+    0.5905951377425391,
+    -0.052139568068853864,
+    0.013673160263486295,
+    -0.03691647864630577,
+    0.09732660653298163,
+    -0.3394662328788498,
+    -0.30685677538541667,
+    -0.24504598907458763,
+    -0.034698524462007344,
+    0.02868032184767538,
+    -0.21217779266454084,
+    -0.1678263169941987,
+    0.3221287889040614,
+    -0.1223055851554907,
+    0.4356604928128464,
+    -0.0502599202236253,
+    0.3979258376211797
+  ],
+  "latents_std": [
+    1.6895524230479284,
+    2.76263727217653,
+    1.7945344281264435,
+    1.6801681847309828,
+    1.6390226546605453,
+    2.7788298348882177,
+    1.7659090095747236,
+    1.6199757612137327,
+    2.6336525640336896,
+    1.8539356672817833,
+    2.5056497896915633,
+    1.811019237886178,
+    1.9579657790720237,
+    1.6685498243529284,
+    1.4922469314453364,
+    3.298670198067373,
+    1.9491804496832168,
+    1.8720003270431442,
+    1.8334080103291832,
+    1.6488070416529093,
+    1.6176957696319716,
+    1.9131449234774398,
+    1.5695245398428617,
+    1.6943659940415912,
+    1.8318420762504692,
+    1.5540637421583379,
+    1.9344930328968526,
+    1.599198216109855,
+    1.718045989838149,
+    1.6307219190837705,
+    1.8661226051202384,
+    1.5613768203168363
+  ]
+}

audio_vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52c59e67ba8de5477c81bfbced0327aabf500f1bfdeefd5ee754529241cb26cb
+size 605429340

benchmarks/astronaut-w4both-w4a16-early-gate.json

@@ -0,0 +1,19 @@
+{
+  "status": "pass",
+  "task": "ref2va",
+  "component": "transformer_ref",
+  "runtime": "packed_w4_dequant_bf16_without_persistent_dequant_cache",
+  "width": 608,
+  "height": 480,
+  "num_frames": 124,
+  "num_inference_steps": 50,
+  "planned_model_evaluations": 49,
+  "completed_model_evaluations": 1,
+  "first_forward_seconds": 212.94,
+  "observed_nvidia_smi_peak_mib": 38577,
+  "checkpoint": "step-001-of-049.pt",
+  "checkpoint_tensor_count": 108,
+  "all_checkpoint_tensors_finite": true,
+  "full_video_status": "not_run",
+  "full_video_reason": "At the measured first-forward rate, a 49-forward Ref2VA smoke would take approximately 2 hours 54 minutes before FP32 VAE decode. This release does not claim an unobserved full Ref2VA result."
+}

benchmarks/astronaut-w4both-w4a16-step001-health.json

@@ -0,0 +1,1200 @@
+{
+  "status": "pass",
+  "checkpoint": "state/checkpoints/astronaut-w4both-w4a16-default50-early/step-001-of-049.pt",
+  "tensor_count": 108,
+  "tensors": [
+    {
+      "path": "root.state.timesteps",
+      "shape": [
+        49
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.0,
+      "max": 0.800000011920929,
+      "mean": 0.14573542773723602
+    },
+    {
+      "path": "root.state.latents",
+      "shape": [
+        16049,
+        96
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": -5.126751899719238,
+      "max": 4.734387397766113,
+      "mean": -0.014345129020512104
+    },
+    {
+      "path": "root.state.audio_latents",
+      "shape": [
+        414,
+        32
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": -3.742875337600708,
+      "max": 4.066157817840576,
+      "mean": -0.015999415889382362
+    },
+    {
+      "path": "root.state.prompt_embeds",
+      "shape": [
+        1,
+        5585,
+        5120
+      ],
+      "dtype": "torch.bfloat16",
+      "finite": true,
+      "min": -160.0,
+      "max": 14336.0,
+      "mean": 0.006581898313015699
+    },
+    {
+      "path": "root.state.row_timestep_plan[0][0]",
+      "shape": [
+        2
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.0,
+      "max": 0.9990000128746033,
+      "mean": 0.49950000643730164
+    },
+    {
+      "path": "root.state.row_timestep_plan[0][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 1.0,
+      "mean": 0.2496371567249298
+    },
+    {
+      "path": "root.state.row_timestep_plan[1][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.0017331242561340332,
+      "max": 0.9990000128746033,
+      "mean": 0.33587655425071716
+    },
+    {
+      "path": "root.state.row_timestep_plan[1][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[2][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.0035335421562194824,
+      "max": 0.9990000128746033,
+      "mean": 0.3388398587703705
+    },
+    {
+      "path": "root.state.row_timestep_plan[2][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[3][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.00540536642074585,
+      "max": 0.9990000128746033,
+      "mean": 0.3418940007686615
+    },
+    {
+      "path": "root.state.row_timestep_plan[3][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[4][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.007352948188781738,
+      "max": 0.9990000128746033,
+      "mean": 0.3450433313846588
+    },
+    {
+      "path": "root.state.row_timestep_plan[4][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[5][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.009380877017974854,
+      "max": 0.9990000128746033,
+      "mean": 0.34829244017601013
+    },
+    {
+      "path": "root.state.row_timestep_plan[5][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[6][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.011494219303131104,
+      "max": 0.9990000128746033,
+      "mean": 0.3516462743282318
+    },
+    {
+      "path": "root.state.row_timestep_plan[6][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[7][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.013698697090148926,
+      "max": 0.9990000128746033,
+      "mean": 0.3551100790500641
+    },
+    {
+      "path": "root.state.row_timestep_plan[7][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[8][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.015999972820281982,
+      "max": 0.9990000128746033,
+      "mean": 0.3586895763874054
+    },
+    {
+      "path": "root.state.row_timestep_plan[8][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[9][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.01840496063232422,
+      "max": 0.9990000128746033,
+      "mean": 0.3623908460140228
+    },
+    {
+      "path": "root.state.row_timestep_plan[9][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[10][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.02092057466506958,
+      "max": 0.9990000128746033,
+      "mean": 0.36622026562690735
+    },
+    {
+      "path": "root.state.row_timestep_plan[10][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[11][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.023554623126983643,
+      "max": 0.9990000128746033,
+      "mean": 0.37018489837646484
+    },
+    {
+      "path": "root.state.row_timestep_plan[11][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[12][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.02631586790084839,
+      "max": 0.9990000128746033,
+      "mean": 0.3742922842502594
+    },
+    {
+      "path": "root.state.row_timestep_plan[12][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[13][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.029213547706604004,
+      "max": 0.9990000128746033,
+      "mean": 0.37855052947998047
+    },
+    {
+      "path": "root.state.row_timestep_plan[13][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[14][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.03225815296173096,
+      "max": 0.9990000128746033,
+      "mean": 0.3829684257507324
+    },
+    {
+      "path": "root.state.row_timestep_plan[14][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[15][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.03546100854873657,
+      "max": 0.9990000128746033,
+      "mean": 0.38755539059638977
+    },
+    {
+      "path": "root.state.row_timestep_plan[15][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[16][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.03883492946624756,
+      "max": 0.9990000128746033,
+      "mean": 0.3923218250274658
+    },
+    {
+      "path": "root.state.row_timestep_plan[16][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[17][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.042394042015075684,
+      "max": 0.9990000128746033,
+      "mean": 0.3972788155078888
+    },
+    {
+      "path": "root.state.row_timestep_plan[17][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[18][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.04615384340286255,
+      "max": 0.9990000128746033,
+      "mean": 0.4024386405944824
+    },
+    {
+      "path": "root.state.row_timestep_plan[18][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[19][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.050131916999816895,
+      "max": 0.9990000128746033,
+      "mean": 0.4078145921230316
+    },
+    {
+      "path": "root.state.row_timestep_plan[19][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[20][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.05434781312942505,
+      "max": 0.9990000128746033,
+      "mean": 0.41342124342918396
+    },
+    {
+      "path": "root.state.row_timestep_plan[20][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[21][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.05882352590560913,
+      "max": 0.9990000128746033,
+      "mean": 0.4192744791507721
+    },
+    {
+      "path": "root.state.row_timestep_plan[21][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[22][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.06358379125595093,
+      "max": 0.9990000128746033,
+      "mean": 0.4253920018672943
+    },
+    {
+      "path": "root.state.row_timestep_plan[22][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[23][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.06865674257278442,
+      "max": 0.9990000128746033,
+      "mean": 0.4317931830883026
+    },
+    {
+      "path": "root.state.row_timestep_plan[23][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[24][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.07407402992248535,
+      "max": 0.9990000128746033,
+      "mean": 0.43849945068359375
+    },
+    {
+      "path": "root.state.row_timestep_plan[24][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[25][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.0798722505569458,
+      "max": 0.9990000128746033,
+      "mean": 0.44553473591804504
+    },
+    {
+      "path": "root.state.row_timestep_plan[25][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[26][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.08609277009963989,
+      "max": 0.9990000128746033,
+      "mean": 0.4529256820678711
+    },
+    {
+      "path": "root.state.row_timestep_plan[26][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[27][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.09278357028961182,
+      "max": 0.9990000128746033,
+      "mean": 0.4607020616531372
+    },
+    {
+      "path": "root.state.row_timestep_plan[27][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[28][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.10000002384185791,
+      "max": 0.9990000128746033,
+      "mean": 0.4688974618911743
+    },
+    {
+      "path": "root.state.row_timestep_plan[28][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[29][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.1078067421913147,
+      "max": 0.9990000128746033,
+      "mean": 0.47754982113838196
+    },
+    {
+      "path": "root.state.row_timestep_plan[29][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[30][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.11627912521362305,
+      "max": 0.9990000128746033,
+      "mean": 0.4867022931575775
+    },
+    {
+      "path": "root.state.row_timestep_plan[30][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[31][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.12550610303878784,
+      "max": 0.9990000128746033,
+      "mean": 0.4964040219783783
+    },
+    {
+      "path": "root.state.row_timestep_plan[31][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[32][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.13559317588806152,
+      "max": 0.9990000128746033,
+      "mean": 0.5067117810249329
+    },
+    {
+      "path": "root.state.row_timestep_plan[32][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[33][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.14666670560836792,
+      "max": 0.9990000128746033,
+      "mean": 0.5176913738250732
+    },
+    {
+      "path": "root.state.row_timestep_plan[33][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[34][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.15887844562530518,
+      "max": 0.9990000128746033,
+      "mean": 0.5294193625450134
+    },
+    {
+      "path": "root.state.row_timestep_plan[34][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[35][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.17241382598876953,
+      "max": 0.9990000128746033,
+      "mean": 0.5419864654541016
+    },
+    {
+      "path": "root.state.row_timestep_plan[35][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[36][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.18750005960464478,
+      "max": 0.9990000128746033,
+      "mean": 0.5555000305175781
+    },
+    {
+      "path": "root.state.row_timestep_plan[36][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[37][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.20441991090774536,
+      "max": 0.9990000128746033,
+      "mean": 0.5700897574424744
+    },
+    {
+      "path": "root.state.row_timestep_plan[37][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[38][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.22352945804595947,
+      "max": 0.9990000128746033,
+      "mean": 0.5859135985374451
+    },
+    {
+      "path": "root.state.row_timestep_plan[38][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[39][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.2452830672264099,
+      "max": 0.9990000128746033,
+      "mean": 0.6031668186187744
+    },
+    {
+      "path": "root.state.row_timestep_plan[39][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[40][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.27027028799057007,
+      "max": 0.9990000128746033,
+      "mean": 0.6220951080322266
+    },
+    {
+      "path": "root.state.row_timestep_plan[40][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[41][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.2992700934410095,
+      "max": 0.9990000128746033,
+      "mean": 0.6430131196975708
+    },
+    {
+      "path": "root.state.row_timestep_plan[41][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[42][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.3333333730697632,
+      "max": 0.9990000128746033,
+      "mean": 0.6663333773612976
+    },
+    {
+      "path": "root.state.row_timestep_plan[42][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[43][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.373913049697876,
+      "max": 0.9990000128746033,
+      "mean": 0.6926103234291077
+    },
+    {
+      "path": "root.state.row_timestep_plan[43][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[44][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.4230769872665405,
+      "max": 0.9990000128746033,
+      "mean": 0.7226132750511169
+    },
+    {
+      "path": "root.state.row_timestep_plan[44][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[45][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.4838709831237793,
+      "max": 0.9990000128746033,
+      "mean": 0.7574481964111328
+    },
+    {
+      "path": "root.state.row_timestep_plan[45][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[46][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.5609756112098694,
+      "max": 0.9990000128746033,
+      "mean": 0.7987797260284424
+    },
+    {
+      "path": "root.state.row_timestep_plan[46][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[47][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.6619718074798584,
+      "max": 0.9990000128746033,
+      "mean": 0.8492547869682312
+    },
+    {
+      "path": "root.state.row_timestep_plan[47][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.row_timestep_plan[48][0]",
+      "shape": [
+        3
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.800000011920929,
+      "max": 0.9990000128746033,
+      "mean": 0.9133921265602112
+    },
+    {
+      "path": "root.state.row_timestep_plan[48][1]",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.5180515050888062
+    },
+    {
+      "path": "root.state.token_tags",
+      "shape": [
+        22048
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 2.0,
+      "mean": 0.041137516498565674
+    },
+    {
+      "path": "root.state.position_ids",
+      "shape": [
+        22048,
+        3
+      ],
+      "dtype": "torch.float64",
+      "finite": true,
+      "min": -2.547236919403076,
+      "max": 5792.0,
+      "mean": 1650.107666015625
+    },
+    {
+      "path": "root.state.video_indices",
+      "shape": [
+        16049
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 5585.0,
+      "max": 22047.0,
+      "mean": 13881.01953125
+    },
+    {
+      "path": "root.state.audio_indices",
+      "shape": [
+        414
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 11089.0,
+      "max": 11502.0,
+      "mean": 11295.5
+    },
+    {
+      "path": "root.state.text_indices",
+      "shape": [
+        5585
+      ],
+      "dtype": "torch.int64",
+      "finite": true,
+      "min": 0.0,
+      "max": 5584.0,
+      "mean": 2792.0
+    },
+    {
+      "path": "root.state.audio_timesteps",
+      "shape": [
+        49
+      ],
+      "dtype": "torch.float32",
+      "finite": true,
+      "min": 0.0,
+      "max": 0.9411764740943909,
+      "mean": 0.3138476312160492
+    }
+  ]
+}

benchmarks/early-gate/native-dynamic-zoom-early.metrics.json

@@ -0,0 +1,57 @@
+{
+  "status": "early_stop",
+  "variant": "orbitquant_w4a4",
+  "bf16_components": [],
+  "component_plan": {
+    "transformer": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/transformer"
+    },
+    "text_encoder": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/text_encoder"
+    },
+    "vae": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/vae"
+    },
+    "audio_vae": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/audio_vae"
+    }
+  },
+  "output_type": "pil",
+  "task": "t2va",
+  "prompt": "Single continuous live-action shot, no cuts. In heavy rain at night, a woman in a yellow raincoat runs directly toward the camera. The camera rapidly dollies backward, then smoothly zooms from a medium close-up to an extreme close-up of her face in the first two seconds. She turns her head left and right while running; wet hair crosses her cheeks; she looks into the lens and says in Russian: «Не отставай, мы почти пришли». Preserve the same face, natural eyes and lips, persistent skin texture, realistic motion blur, neon reflections, rain ambience.",
+  "seed": 42,
+  "height": 480,
+  "width": 608,
+  "num_frames": 124,
+  "fps": 24,
+  "num_inference_steps": 50,
+  "model_evaluations": 49,
+  "planned_denoise_steps": 1,
+  "checkpoint_dir": "/workspace/orbitquant-h3-perf/runs/native-default50/checkpoints",
+  "checkpoint_policy": "atomic_block_state_after_each_scheduler_step",
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "pid": 6935,
+  "h3_dtype_views": 50,
+  "transformer_runtime_mode": "auto_fused",
+  "transformer_runtime_mode_modules": 300,
+  "w4a4_int8_weight_cache_modules": 0,
+  "load_seconds": 1.487323367036879,
+  "offload": {
+    "mode": "manual_stage_offload",
+    "conditioner": "cuda_then_cpu",
+    "transformer": "cpu_then_cuda",
+    "vae": "cpu",
+    "audio_vae": "cpu"
+  },
+  "transformer_dequant_cache_disabled_modules": 0,
+  "move_to_cuda_seconds": 1.5776299671269953,
+  "cuda_after_load_bytes": 19951415808,
+  "completed_denoise_steps": 1,
+  "total_denoise_steps": 49,
+  "generation_seconds": 12.247872655745596,
+  "cuda_generation_peak_bytes": 46354671104
+}

benchmarks/early-gate/native-vs-w4a16-step001.json

@@ -0,0 +1,38 @@
+{
+  "status": "compared",
+  "completed_step": 1,
+  "total_steps": 49,
+  "matched_metadata": {
+    "task": "t2va",
+    "prompt": "Single continuous live-action shot, no cuts. In heavy rain at night, a woman in a yellow raincoat runs directly toward the camera. The camera rapidly dollies backward, then smoothly zooms from a medium close-up to an extreme close-up of her face in the first two seconds. She turns her head left and right while running; wet hair crosses her cheeks; she looks into the lens and says in Russian: «Не отставай, мы почти пришли». Preserve the same face, natural eyes and lips, persistent skin texture, realistic motion blur, neon reflections, rain ambience.",
+    "seed": 42,
+    "height": 480,
+    "width": 608,
+    "num_frames": 124,
+    "num_inference_steps": 50
+  },
+  "video_latents": {
+    "shape": [
+      10545,
+      96
+    ],
+    "rmse": 0.0004155437054578215,
+    "reference_rms": 0.998328447341919,
+    "nrmse": 0.00041623947165306144,
+    "cosine_similarity": 1.0000184456056993,
+    "max_abs_error": 0.002462148666381836
+  },
+  "audio_latents": {
+    "shape": [
+      414,
+      32
+    ],
+    "rmse": 0.000725507561583072,
+    "reference_rms": 0.9890910387039185,
+    "nrmse": 0.0007335093870972281,
+    "cosine_similarity": 0.9999999635822313,
+    "max_abs_error": 0.0029428284615278244
+  },
+  "reference_checkpoint": "/workspace/orbitquant-h3-perf/runs/dequant-control/checkpoints/step-001-of-049.pt",
+  "candidate_checkpoint": "/workspace/orbitquant-h3-perf/runs/native-default50/checkpoints/step-001-of-049.pt"
+}

benchmarks/early-gate/w4a16-dynamic-zoom-early.metrics.json

@@ -0,0 +1,57 @@
+{
+  "status": "early_stop",
+  "variant": "orbitquant_w4a4_text_w4a16_transformer",
+  "bf16_components": [],
+  "component_plan": {
+    "transformer": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/transformer"
+    },
+    "text_encoder": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/text_encoder"
+    },
+    "vae": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/vae"
+    },
+    "audio_vae": {
+      "source": "release",
+      "path": "/workspace/orbitquant-h3-perf/model/audio_vae"
+    }
+  },
+  "output_type": "pil",
+  "task": "t2va",
+  "prompt": "Single continuous live-action shot, no cuts. In heavy rain at night, a woman in a yellow raincoat runs directly toward the camera. The camera rapidly dollies backward, then smoothly zooms from a medium close-up to an extreme close-up of her face in the first two seconds. She turns her head left and right while running; wet hair crosses her cheeks; she looks into the lens and says in Russian: «Не отставай, мы почти пришли». Preserve the same face, natural eyes and lips, persistent skin texture, realistic motion blur, neon reflections, rain ambience.",
+  "seed": 42,
+  "height": 480,
+  "width": 608,
+  "num_frames": 124,
+  "fps": 24,
+  "num_inference_steps": 50,
+  "model_evaluations": 49,
+  "planned_denoise_steps": 1,
+  "checkpoint_dir": "/workspace/orbitquant-h3-perf/runs/dequant-control/checkpoints",
+  "checkpoint_policy": "atomic_block_state_after_each_scheduler_step",
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "pid": 7396,
+  "h3_dtype_views": 50,
+  "transformer_runtime_mode": "dequant_bf16",
+  "transformer_runtime_mode_modules": 300,
+  "w4a4_int8_weight_cache_modules": 0,
+  "load_seconds": 1.3163598240353167,
+  "offload": {
+    "mode": "manual_stage_offload",
+    "conditioner": "cuda_then_cpu",
+    "transformer": "cpu_then_cuda",
+    "vae": "cpu",
+    "audio_vae": "cpu"
+  },
+  "transformer_dequant_cache_disabled_modules": 300,
+  "move_to_cuda_seconds": 1.5451071108691394,
+  "cuda_after_load_bytes": 19951415808,
+  "completed_denoise_steps": 1,
+  "total_denoise_steps": 49,
+  "generation_seconds": 9.525488262996078,
+  "cuda_generation_peak_bytes": 46354364928
+}

benchmarks/example_metrics.csv

@@ -0,0 +1,4 @@
+id,role,task,seed,variant,generation_seconds,decode_seconds,load_seconds,cuda_generation_peak_bytes,cuda_decode_peak_bytes,h264_bytes,hevc_bytes
+source-bf16-control,source control,t2va,74001,controlled_bf16_ablation,451.5754231447354,19.563463208731264,1.5101884552277625,68463336960,12139359744,635349,375788
+w4-text-control,component isolation control,t2va,74001,controlled_bf16_ablation,381.5656377780251,13.154935840982944,2.5672516343183815,68464447488,12139359744,753550,449021
+w4a4-native-t2va,final native W4A4 T2VA recipe,t2va,42,orbitquant_w4a4,114.6722432100214,4.709521219134331,1.4625724367797375,46359644160,12139359744,815089,474343

benchmarks/example_metrics.json

@@ -0,0 +1,196 @@
+[
+  {
+    "id": "source-bf16-control",
+    "title": "Source BF16 control — dynamic zoom and dialogue",
+    "role": "source control",
+    "task": "t2va",
+    "seed": 74001,
+    "variant": "controlled_bf16_ablation",
+    "generation_seconds": 451.5754231447354,
+    "decode_seconds": 19.563463208731264,
+    "load_seconds": 1.5101884552277625,
+    "cuda_generation_peak_bytes": 68463336960,
+    "cuda_decode_peak_bytes": 12139359744,
+    "offload": {
+      "mode": "components_manager_auto_cpu_offload",
+      "device": "cuda",
+      "memory_reserve_margin": "64GB"
+    },
+    "quality_gate": {
+      "status": "pass",
+      "model_evaluations": 49,
+      "reviewed_frame_count": 26
+    },
+    "manual_review": {
+      "status": "pass",
+      "review_type": "manual_visual_temporal",
+      "reviewed_frame_indices": [
+        0,
+        5,
+        10,
+        15,
+        20,
+        25,
+        30,
+        35,
+        40,
+        45,
+        50,
+        55,
+        60,
+        65,
+        70,
+        75,
+        80,
+        85,
+        90,
+        95,
+        100,
+        105,
+        110,
+        115,
+        120,
+        123
+      ],
+      "prompt_subject_recognizable": true,
+      "coherent_motion": true,
+      "repeating_tile_artifacts": false,
+      "motion_ghosting": false,
+      "texture_breakup": false,
+      "face_integrity": true,
+      "full_resolution_frame_reviewed": true,
+      "notes": "Dense every-fifth-frame and 6 FPS tail review preserves one face, aligned eyes and lips, and persistent wet-skin texture. Fast running frames contain ordinary motion blur that resolves in adjacent frames. This review is visual only; generated audio still requires a listening gate."
+    },
+    "h264_bytes": 635349,
+    "hevc_bytes": 375788
+  },
+  {
+    "id": "w4-text-control",
+    "title": "OrbitQuant W4A4 text encoder, BF16 denoiser",
+    "role": "component isolation control",
+    "task": "t2va",
+    "seed": 74001,
+    "variant": "controlled_bf16_ablation",
+    "generation_seconds": 381.5656377780251,
+    "decode_seconds": 13.154935840982944,
+    "load_seconds": 2.5672516343183815,
+    "cuda_generation_peak_bytes": 68464447488,
+    "cuda_decode_peak_bytes": 12139359744,
+    "offload": {
+      "mode": "components_manager_auto_cpu_offload",
+      "device": "cuda",
+      "memory_reserve_margin": "64GB"
+    },
+    "quality_gate": {
+      "status": "pass",
+      "model_evaluations": 49,
+      "reviewed_frame_count": 26
+    },
+    "manual_review": {
+      "status": "pass",
+      "review_type": "manual_visual_temporal",
+      "reviewed_frame_indices": [
+        0,
+        5,
+        10,
+        15,
+        20,
+        25,
+        30,
+        35,
+        40,
+        45,
+        50,
+        55,
+        60,
+        65,
+        70,
+        75,
+        80,
+        85,
+        90,
+        95,
+        100,
+        105,
+        110,
+        115,
+        120,
+        123
+      ],
+      "prompt_subject_recognizable": true,
+      "coherent_motion": true,
+      "repeating_tile_artifacts": false,
+      "motion_ghosting": false,
+      "texture_breakup": false,
+      "face_integrity": true,
+      "full_resolution_frame_reviewed": true,
+      "audio_listening_status": "pass",
+      "audio_reviewed_by": "user",
+      "notes": "Dense every-fifth-frame and 6 FPS tail review preserves one face, aligned eyes and lips, and persistent wet-skin texture. Full-resolution frames 80 and 123 were inspected directly. The user reviewed the embedded video with audio and judged it ideal."
+    },
+    "h264_bytes": 753550,
+    "hevc_bytes": 449021
+  },
+  {
+    "id": "w4a4-native-t2va",
+    "title": "Final native W4A4 text + W4A4 T2VA transformer",
+    "role": "final native W4A4 T2VA recipe",
+    "task": "t2va",
+    "seed": 42,
+    "variant": "orbitquant_w4a4",
+    "generation_seconds": 114.6722432100214,
+    "decode_seconds": 4.709521219134331,
+    "load_seconds": 1.4625724367797375,
+    "cuda_generation_peak_bytes": 46359644160,
+    "cuda_decode_peak_bytes": 12139359744,
+    "offload": {
+      "mode": "manual_stage_offload",
+      "conditioner": "cuda_then_cpu",
+      "transformer": "cpu_then_cuda",
+      "vae": "cpu",
+      "audio_vae": "cpu"
+    },
+    "quality_gate": {
+      "status": "pass",
+      "model_evaluations": 49,
+      "reviewed_frame_count": 19
+    },
+    "manual_review": {
+      "status": "pass",
+      "review_type": "manual_visual_temporal_and_audio_signal",
+      "reviewed_frame_indices": [
+        0,
+        24,
+        48,
+        60,
+        72,
+        80,
+        84,
+        88,
+        90,
+        92,
+        96,
+        100,
+        104,
+        108,
+        110,
+        112,
+        116,
+        120,
+        123
+      ],
+      "prompt_subject_recognizable": true,
+      "coherent_motion": true,
+      "repeating_tile_artifacts": false,
+      "motion_ghosting": false,
+      "texture_breakup": false,
+      "face_integrity": true,
+      "full_resolution_frame_reviewed": true,
+      "audio_signal_integrity": true,
+      "audio_user_listening_status": "pending",
+      "notes": "Full-timeline and dense late-motion grids plus full-resolution frames 60, 90, 110, and 123 preserve one coherent face, aligned eyes, connected lip motion, teeth, rain, and skin texture without the earlier grid, doubled contour, ghosting, or breakup. The HEVC/AAC file decodes to 124 frames at 608x480. Audio has no NaN/Inf samples or clipping, peaks at -0.27 dBFS, and has no persistent high-frequency narrow-band tone; energy above 8 kHz is 1.42%, below the accepted W4A16 control's 2.99%."
+    },
+    "h264_bytes": 815089,
+    "hevc_bytes": 474343
+  }
+]

benchmarks/media_probe.json

@@ -0,0 +1,197 @@
+[
+  {
+    "id": "source-bf16-control",
+    "h264": {
+      "programs": [],
+      "streams": [
+        {
+          "index": 0,
+          "codec_name": "h264",
+          "codec_type": "video",
+          "codec_tag_string": "avc1",
+          "width": 608,
+          "height": 480,
+          "r_frame_rate": "24/1",
+          "nb_frames": "124",
+          "nb_read_frames": "124"
+        },
+        {
+          "index": 1,
+          "codec_name": "aac",
+          "codec_type": "audio",
+          "codec_tag_string": "mp4a",
+          "sample_rate": "32000",
+          "channels": 2,
+          "r_frame_rate": "0/0",
+          "nb_frames": "163",
+          "nb_read_frames": "162"
+        }
+      ],
+      "format": {
+        "duration": "5.175000",
+        "size": "635349"
+      }
+    },
+    "hevc": {
+      "programs": [],
+      "streams": [
+        {
+          "index": 0,
+          "codec_name": "hevc",
+          "codec_type": "video",
+          "codec_tag_string": "hvc1",
+          "width": 608,
+          "height": 480,
+          "r_frame_rate": "24/1",
+          "nb_frames": "124",
+          "nb_read_frames": "124"
+        },
+        {
+          "index": 1,
+          "codec_name": "aac",
+          "codec_type": "audio",
+          "codec_tag_string": "mp4a",
+          "sample_rate": "32000",
+          "channels": 2,
+          "r_frame_rate": "0/0",
+          "nb_frames": "163",
+          "nb_read_frames": "162"
+        }
+      ],
+      "format": {
+        "duration": "5.175000",
+        "size": "375788"
+      }
+    }
+  },
+  {
+    "id": "w4-text-control",
+    "h264": {
+      "programs": [],
+      "streams": [
+        {
+          "index": 0,
+          "codec_name": "h264",
+          "codec_type": "video",
+          "codec_tag_string": "avc1",
+          "width": 608,
+          "height": 480,
+          "r_frame_rate": "24/1",
+          "nb_frames": "124",
+          "nb_read_frames": "124"
+        },
+        {
+          "index": 1,
+          "codec_name": "aac",
+          "codec_type": "audio",
+          "codec_tag_string": "mp4a",
+          "sample_rate": "32000",
+          "channels": 2,
+          "r_frame_rate": "0/0",
+          "nb_frames": "163",
+          "nb_read_frames": "162"
+        }
+      ],
+      "format": {
+        "duration": "5.175000",
+        "size": "753550"
+      }
+    },
+    "hevc": {
+      "programs": [],
+      "streams": [
+        {
+          "index": 0,
+          "codec_name": "hevc",
+          "codec_type": "video",
+          "codec_tag_string": "hvc1",
+          "width": 608,
+          "height": 480,
+          "r_frame_rate": "24/1",
+          "nb_frames": "124",
+          "nb_read_frames": "124"
+        },
+        {
+          "index": 1,
+          "codec_name": "aac",
+          "codec_type": "audio",
+          "codec_tag_string": "mp4a",
+          "sample_rate": "32000",
+          "channels": 2,
+          "r_frame_rate": "0/0",
+          "nb_frames": "163",
+          "nb_read_frames": "162"
+        }
+      ],
+      "format": {
+        "duration": "5.175000",
+        "size": "449021"
+      }
+    }
+  },
+  {
+    "id": "w4a4-native-t2va",
+    "h264": {
+      "programs": [],
+      "streams": [
+        {
+          "index": 0,
+          "codec_name": "h264",
+          "codec_type": "video",
+          "codec_tag_string": "avc1",
+          "width": 608,
+          "height": 480,
+          "r_frame_rate": "24/1",
+          "nb_frames": "124",
+          "nb_read_frames": "124"
+        },
+        {
+          "index": 1,
+          "codec_name": "aac",
+          "codec_type": "audio",
+          "codec_tag_string": "mp4a",
+          "sample_rate": "32000",
+          "channels": 2,
+          "r_frame_rate": "0/0",
+          "nb_frames": "163",
+          "nb_read_frames": "162"
+        }
+      ],
+      "format": {
+        "duration": "5.184000",
+        "size": "815089"
+      }
+    },
+    "hevc": {
+      "programs": [],
+      "streams": [
+        {
+          "index": 0,
+          "codec_name": "hevc",
+          "codec_type": "video",
+          "codec_tag_string": "hvc1",
+          "width": 608,
+          "height": 480,
+          "r_frame_rate": "24/1",
+          "nb_frames": "124",
+          "nb_read_frames": "124"
+        },
+        {
+          "index": 1,
+          "codec_name": "aac",
+          "codec_type": "audio",
+          "codec_tag_string": "mp4a",
+          "sample_rate": "32000",
+          "channels": 2,
+          "r_frame_rate": "0/0",
+          "nb_frames": "163",
+          "nb_read_frames": "162"
+        }
+      ],
+      "format": {
+        "duration": "5.184000",
+        "size": "474343"
+      }
+    }
+  }
+]

benchmarks/profiles/native-full-forward-bf16-load.json

@@ -0,0 +1,4629 @@
+{
+  "activation_backends": {
+    "triton_cuda_packed_w4": 300
+  },
+  "audio_tokens": 207,
+  "capability": [
+    12,
+    0
+  ],
+  "cuda": "12.8",
+  "cuda_after_load_bytes": 18328776192,
+  "cuda_forward_ms": 2867.617919921875,
+  "cuda_peak_bytes": 46381307904,
+  "effective_runtime_modes": {
+    "native_packed_matmul": 300
+  },
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "h3_dtype_views": 50,
+  "load_dtype": "bfloat16",
+  "load_seconds": 0.28624553605914116,
+  "model_dtype": "torch.bfloat16",
+  "move_to_cuda_seconds": 1.1773170102387667,
+  "num_blocks": 50,
+  "orbitquant_linear_count": 300,
+  "orbitquant_linear_cuda_ms": 1077.578048467636,
+  "orbitquant_linear_inputs": {
+    "transformer_blocks.0.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    }
+  },
+  "orbitquant_shape_cuda_ms": {
+    "14336x5376": {
+      "calls": 50,
+      "total_ms": 229.30192136764526
+    },
+    "5376x28672": {
+      "calls": 50,
+      "total_ms": 352.2663993835449
+    },
+    "5376x7168": {
+      "calls": 150,
+      "total_ms": 371.99267172813416
+    },
+    "7168x5376": {
+      "calls": 50,
+      "total_ms": 124.01705598831177
+    }
+  },
+  "orbitquant_shapes": {
+    "14336x5376": 50,
+    "5376x28672": 50,
+    "5376x7168": 150,
+    "7168x5376": 50
+  },
+  "output_finite": [
+    true,
+    true
+  ],
+  "output_shapes": [
+    [
+      1,
+      10545,
+      96
+    ],
+    [
+      1,
+      207,
+      32
+    ]
+  ],
+  "runtime_mode": "native_packed_matmul",
+  "slowest_orbitquant_linears": [
+    {
+      "cuda_ms": 18.641632080078125,
+      "name": "transformer_blocks.0.attn.to_q",
+      "shape": "5376x7168"
+    },
+    {
+      "cuda_ms": 9.849472045898438,
+      "name": "transformer_blocks.0.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 8.686016082763672,
+      "name": "transformer_blocks.0.ff.net.2",
+      "shape": "14336x5376"
+    },
+    {
+      "cuda_ms": 7.292928218841553,
+      "name": "transformer_blocks.0.attn.to_out.0",
+      "shape": "7168x5376"
+    },
+    {
+      "cuda_ms": 7.155712127685547,
+      "name": "transformer_blocks.14.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.124512195587158,
+      "name": "transformer_blocks.24.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.1244797706604,
+      "name": "transformer_blocks.2.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.116352081298828,
+      "name": "transformer_blocks.7.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.115295886993408,
+      "name": "transformer_blocks.9.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.111199855804443,
+      "name": "transformer_blocks.5.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.111199855804443,
+      "name": "transformer_blocks.11.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.103007793426514,
+      "name": "transformer_blocks.19.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.101952075958252,
+      "name": "transformer_blocks.12.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.098879814147949,
+      "name": "transformer_blocks.25.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.094880104064941,
+      "name": "transformer_blocks.6.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0866241455078125,
+      "name": "transformer_blocks.4.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.085055828094482,
+      "name": "transformer_blocks.17.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.08355188369751,
+      "name": "transformer_blocks.18.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.080480098724365,
+      "name": "transformer_blocks.30.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.074431896209717,
+      "name": "transformer_blocks.29.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.068192005157471,
+      "name": "transformer_blocks.10.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.064576148986816,
+      "name": "transformer_blocks.15.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.062047958374023,
+      "name": "transformer_blocks.27.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0609917640686035,
+      "name": "transformer_blocks.1.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.059999942779541,
+      "name": "transformer_blocks.3.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.059999942779541,
+      "name": "transformer_blocks.34.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.059967994689941,
+      "name": "transformer_blocks.20.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.058976173400879,
+      "name": "transformer_blocks.32.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.056896209716797,
+      "name": "transformer_blocks.13.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.055903911590576,
+      "name": "transformer_blocks.46.ff.net.0.proj",
+      "shape": "5376x28672"
+    }
+  ],
+  "status": "pass",
+  "text_tokens": 512,
+  "torch": "2.8.0+cu128",
+  "transformer": "/workspace/orbitquant-h3-perf/model/transformer",
+  "transformer_block_cuda_ms": [
+    {
+      "cuda_ms": 309.7416687011719,
+      "name": "transformer_blocks.0"
+    },
+    {
+      "cuda_ms": 48.530784606933594,
+      "name": "transformer_blocks.1"
+    },
+    {
+      "cuda_ms": 48.752193450927734,
+      "name": "transformer_blocks.2"
+    },
+    {
+      "cuda_ms": 48.56070327758789,
+      "name": "transformer_blocks.3"
+    },
+    {
+      "cuda_ms": 48.65910339355469,
+      "name": "transformer_blocks.4"
+    },
+    {
+      "cuda_ms": 48.712257385253906,
+      "name": "transformer_blocks.5"
+    },
+    {
+      "cuda_ms": 48.625247955322266,
+      "name": "transformer_blocks.6"
+    },
+    {
+      "cuda_ms": 48.70915222167969,
+      "name": "transformer_blocks.7"
+    },
+    {
+      "cuda_ms": 48.56073760986328,
+      "name": "transformer_blocks.8"
+    },
+    {
+      "cuda_ms": 48.7092170715332,
+      "name": "transformer_blocks.9"
+    },
+    {
+      "cuda_ms": 48.58531188964844,
+      "name": "transformer_blocks.10"
+    },
+    {
+      "cuda_ms": 48.66614532470703,
+      "name": "transformer_blocks.11"
+    },
+    {
+      "cuda_ms": 48.691776275634766,
+      "name": "transformer_blocks.12"
+    },
+    {
+      "cuda_ms": 48.55561447143555,
+      "name": "transformer_blocks.13"
+    },
+    {
+      "cuda_ms": 48.70761489868164,
+      "name": "transformer_blocks.14"
+    },
+    {
+      "cuda_ms": 48.58272171020508,
+      "name": "transformer_blocks.15"
+    },
+    {
+      "cuda_ms": 48.22224044799805,
+      "name": "transformer_blocks.16"
+    },
+    {
+      "cuda_ms": 48.6544303894043,
+      "name": "transformer_blocks.17"
+    },
+    {
+      "cuda_ms": 48.637569427490234,
+      "name": "transformer_blocks.18"
+    },
+    {
+      "cuda_ms": 48.6978874206543,
+      "name": "transformer_blocks.19"
+    },
+    {
+      "cuda_ms": 48.58736038208008,
+      "name": "transformer_blocks.20"
+    },
+    {
+      "cuda_ms": 48.09062576293945,
+      "name": "transformer_blocks.21"
+    },
+    {
+      "cuda_ms": 48.501312255859375,
+      "name": "transformer_blocks.22"
+    },
+    {
+      "cuda_ms": 48.043521881103516,
+      "name": "transformer_blocks.23"
+    },
+    {
+      "cuda_ms": 48.79324722290039,
+      "name": "transformer_blocks.24"
+    },
+    {
+      "cuda_ms": 48.69996643066406,
+      "name": "transformer_blocks.25"
+    },
+    {
+      "cuda_ms": 48.07846450805664,
+      "name": "transformer_blocks.26"
+    },
+    {
+      "cuda_ms": 48.556575775146484,
+      "name": "transformer_blocks.27"
+    },
+    {
+      "cuda_ms": 48.14499282836914,
+      "name": "transformer_blocks.28"
+    },
+    {
+      "cuda_ms": 48.63030242919922,
+      "name": "transformer_blocks.29"
+    },
+    {
+      "cuda_ms": 48.61299133300781,
+      "name": "transformer_blocks.30"
+    },
+    {
+      "cuda_ms": 48.09686279296875,
+      "name": "transformer_blocks.31"
+    },
+    {
+      "cuda_ms": 48.560672760009766,
+      "name": "transformer_blocks.32"
+    },
+    {
+      "cuda_ms": 48.092735290527344,
+      "name": "transformer_blocks.33"
+    },
+    {
+      "cuda_ms": 48.55769729614258,
+      "name": "transformer_blocks.34"
+    },
+    {
+      "cuda_ms": 48.0081901550293,
+      "name": "transformer_blocks.35"
+    },
+    {
+      "cuda_ms": 48.5715217590332,
+      "name": "transformer_blocks.36"
+    },
+    {
+      "cuda_ms": 48.09116744995117,
+      "name": "transformer_blocks.37"
+    },
+    {
+      "cuda_ms": 47.97200012207031,
+      "name": "transformer_blocks.38"
+    },
+    {
+      "cuda_ms": 48.52592086791992,
+      "name": "transformer_blocks.39"
+    },
+    {
+      "cuda_ms": 47.92879867553711,
+      "name": "transformer_blocks.40"
+    },
+    {
+      "cuda_ms": 48.56079864501953,
+      "name": "transformer_blocks.41"
+    },
+    {
+      "cuda_ms": 48.176639556884766,
+      "name": "transformer_blocks.42"
+    },
+    {
+      "cuda_ms": 48.074241638183594,
+      "name": "transformer_blocks.43"
+    },
+    {
+      "cuda_ms": 48.551456451416016,
+      "name": "transformer_blocks.44"
+    },
+    {
+      "cuda_ms": 47.84288024902344,
+      "name": "transformer_blocks.45"
+    },
+    {
+      "cuda_ms": 48.65087890625,
+      "name": "transformer_blocks.46"
+    },
+    {
+      "cuda_ms": 47.98934555053711,
+      "name": "transformer_blocks.47"
+    },
+    {
+      "cuda_ms": 48.15216064453125,
+      "name": "transformer_blocks.48"
+    },
+    {
+      "cuda_ms": 48.495201110839844,
+      "name": "transformer_blocks.49"
+    }
+  ],
+  "video_tokens": 10545,
+  "wall_seconds": 2.8687175642699003
+}

benchmarks/profiles/native-full-forward-fp32-load.json

@@ -0,0 +1,424 @@
+{
+  "activation_backends": {
+    "triton_cuda": 300
+  },
+  "audio_tokens": 207,
+  "capability": [
+    12,
+    0
+  ],
+  "cuda": "12.8",
+  "cuda_after_load_bytes": 19996112896,
+  "cuda_forward_ms": 114115.6328125,
+  "cuda_peak_bytes": 49839283712,
+  "effective_runtime_modes": {
+    "native_packed_matmul": 300
+  },
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "h3_dtype_views": 50,
+  "load_seconds": 0.7637474560178816,
+  "move_to_cuda_seconds": 1.5632987627759576,
+  "orbitquant_linear_count": 300,
+  "orbitquant_linear_cuda_ms": 109206.42547607422,
+  "orbitquant_shape_cuda_ms": {
+    "14336x5376": {
+      "calls": 50,
+      "total_ms": 22013.64224243164
+    },
+    "5376x28672": {
+      "calls": 50,
+      "total_ms": 43292.70489501953
+    },
+    "5376x7168": {
+      "calls": 150,
+      "total_ms": 32823.50001525879
+    },
+    "7168x5376": {
+      "calls": 50,
+      "total_ms": 11076.578323364258
+    }
+  },
+  "orbitquant_shapes": {
+    "14336x5376": 50,
+    "5376x28672": 50,
+    "5376x7168": 150,
+    "7168x5376": 50
+  },
+  "output_finite": [
+    true,
+    true
+  ],
+  "output_shapes": [
+    [
+      1,
+      10545,
+      96
+    ],
+    [
+      1,
+      207,
+      32
+    ]
+  ],
+  "runtime_mode": "native_packed_matmul",
+  "slowest_orbitquant_linears": [
+    {
+      "cuda_ms": 913.251953125,
+      "name": "transformer_blocks.2.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 911.2760620117188,
+      "name": "transformer_blocks.0.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 901.734375,
+      "name": "transformer_blocks.1.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 901.5783081054688,
+      "name": "transformer_blocks.3.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 868.2454833984375,
+      "name": "transformer_blocks.4.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 865.6199951171875,
+      "name": "transformer_blocks.5.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.7640380859375,
+      "name": "transformer_blocks.6.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.7313232421875,
+      "name": "transformer_blocks.7.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.56640625,
+      "name": "transformer_blocks.37.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.19775390625,
+      "name": "transformer_blocks.35.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.1362915039062,
+      "name": "transformer_blocks.47.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0994262695312,
+      "name": "transformer_blocks.39.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0728149414062,
+      "name": "transformer_blocks.43.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0718383789062,
+      "name": "transformer_blocks.40.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0676879882812,
+      "name": "transformer_blocks.33.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0646362304688,
+      "name": "transformer_blocks.45.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0635986328125,
+      "name": "transformer_blocks.49.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.060546875,
+      "name": "transformer_blocks.36.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0595092773438,
+      "name": "transformer_blocks.42.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0595092773438,
+      "name": "transformer_blocks.46.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0513305664062,
+      "name": "transformer_blocks.44.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0328979492188,
+      "name": "transformer_blocks.24.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0308227539062,
+      "name": "transformer_blocks.17.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0308227539062,
+      "name": "transformer_blocks.20.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0298461914062,
+      "name": "transformer_blocks.41.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0267333984375,
+      "name": "transformer_blocks.16.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0216064453125,
+      "name": "transformer_blocks.34.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0206298828125,
+      "name": "transformer_blocks.29.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0175170898438,
+      "name": "transformer_blocks.22.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 862.0165405273438,
+      "name": "transformer_blocks.21.ff.net.0.proj",
+      "shape": "5376x28672"
+    }
+  ],
+  "status": "pass",
+  "text_tokens": 512,
+  "torch": "2.8.0+cu128",
+  "transformer": "/workspace/orbitquant-h3-perf/model/transformer",
+  "transformer_block_cuda_ms": [
+    {
+      "cuda_ms": 3487.91259765625,
+      "name": "transformer_blocks.0"
+    },
+    {
+      "cuda_ms": 2355.49658203125,
+      "name": "transformer_blocks.1"
+    },
+    {
+      "cuda_ms": 2366.400146484375,
+      "name": "transformer_blocks.2"
+    },
+    {
+      "cuda_ms": 2352.244384765625,
+      "name": "transformer_blocks.3"
+    },
+    {
+      "cuda_ms": 2286.18408203125,
+      "name": "transformer_blocks.4"
+    },
+    {
+      "cuda_ms": 2254.422607421875,
+      "name": "transformer_blocks.5"
+    },
+    {
+      "cuda_ms": 2250.595947265625,
+      "name": "transformer_blocks.6"
+    },
+    {
+      "cuda_ms": 2248.635986328125,
+      "name": "transformer_blocks.7"
+    },
+    {
+      "cuda_ms": 2246.948486328125,
+      "name": "transformer_blocks.8"
+    },
+    {
+      "cuda_ms": 2246.48046875,
+      "name": "transformer_blocks.9"
+    },
+    {
+      "cuda_ms": 2246.338134765625,
+      "name": "transformer_blocks.10"
+    },
+    {
+      "cuda_ms": 2246.347412109375,
+      "name": "transformer_blocks.11"
+    },
+    {
+      "cuda_ms": 2246.5205078125,
+      "name": "transformer_blocks.12"
+    },
+    {
+      "cuda_ms": 2246.44677734375,
+      "name": "transformer_blocks.13"
+    },
+    {
+      "cuda_ms": 2246.46630859375,
+      "name": "transformer_blocks.14"
+    },
+    {
+      "cuda_ms": 2246.548095703125,
+      "name": "transformer_blocks.15"
+    },
+    {
+      "cuda_ms": 2246.48046875,
+      "name": "transformer_blocks.16"
+    },
+    {
+      "cuda_ms": 2246.6708984375,
+      "name": "transformer_blocks.17"
+    },
+    {
+      "cuda_ms": 2246.313720703125,
+      "name": "transformer_blocks.18"
+    },
+    {
+      "cuda_ms": 2246.57373046875,
+      "name": "transformer_blocks.19"
+    },
+    {
+      "cuda_ms": 2246.4384765625,
+      "name": "transformer_blocks.20"
+    },
+    {
+      "cuda_ms": 2246.5458984375,
+      "name": "transformer_blocks.21"
+    },
+    {
+      "cuda_ms": 2246.38330078125,
+      "name": "transformer_blocks.22"
+    },
+    {
+      "cuda_ms": 2246.49365234375,
+      "name": "transformer_blocks.23"
+    },
+    {
+      "cuda_ms": 2246.5146484375,
+      "name": "transformer_blocks.24"
+    },
+    {
+      "cuda_ms": 2246.240966796875,
+      "name": "transformer_blocks.25"
+    },
+    {
+      "cuda_ms": 2246.17626953125,
+      "name": "transformer_blocks.26"
+    },
+    {
+      "cuda_ms": 2246.2880859375,
+      "name": "transformer_blocks.27"
+    },
+    {
+      "cuda_ms": 2246.201904296875,
+      "name": "transformer_blocks.28"
+    },
+    {
+      "cuda_ms": 2246.2490234375,
+      "name": "transformer_blocks.29"
+    },
+    {
+      "cuda_ms": 2246.255126953125,
+      "name": "transformer_blocks.30"
+    },
+    {
+      "cuda_ms": 2246.263427734375,
+      "name": "transformer_blocks.31"
+    },
+    {
+      "cuda_ms": 2246.54296875,
+      "name": "transformer_blocks.32"
+    },
+    {
+      "cuda_ms": 2246.833740234375,
+      "name": "transformer_blocks.33"
+    },
+    {
+      "cuda_ms": 2246.46826171875,
+      "name": "transformer_blocks.34"
+    },
+    {
+      "cuda_ms": 2246.546630859375,
+      "name": "transformer_blocks.35"
+    },
+    {
+      "cuda_ms": 2246.568359375,
+      "name": "transformer_blocks.36"
+    },
+    {
+      "cuda_ms": 2247.191162109375,
+      "name": "transformer_blocks.37"
+    },
+    {
+      "cuda_ms": 2246.4765625,
+      "name": "transformer_blocks.38"
+    },
+    {
+      "cuda_ms": 2246.99853515625,
+      "name": "transformer_blocks.39"
+    },
+    {
+      "cuda_ms": 2246.68359375,
+      "name": "transformer_blocks.40"
+    },
+    {
+      "cuda_ms": 2246.66064453125,
+      "name": "transformer_blocks.41"
+    },
+    {
+      "cuda_ms": 2246.555419921875,
+      "name": "transformer_blocks.42"
+    },
+    {
+      "cuda_ms": 2246.562255859375,
+      "name": "transformer_blocks.43"
+    },
+    {
+      "cuda_ms": 2246.51025390625,
+      "name": "transformer_blocks.44"
+    },
+    {
+      "cuda_ms": 2246.63623046875,
+      "name": "transformer_blocks.45"
+    },
+    {
+      "cuda_ms": 2246.504150390625,
+      "name": "transformer_blocks.46"
+    },
+    {
+      "cuda_ms": 2246.570556640625,
+      "name": "transformer_blocks.47"
+    },
+    {
+      "cuda_ms": 2246.426025390625,
+      "name": "transformer_blocks.48"
+    },
+    {
+      "cuda_ms": 2246.447998046875,
+      "name": "transformer_blocks.49"
+    }
+  ],
+  "video_tokens": 10545,
+  "wall_seconds": 114.1175869773142
+}

benchmarks/profiles/native-full-forward-package-default.json

@@ -0,0 +1,4629 @@
+{
+  "activation_backends": {
+    "triton_cuda_packed_w4": 300
+  },
+  "audio_tokens": 207,
+  "capability": [
+    12,
+    0
+  ],
+  "cuda": "12.8",
+  "cuda_after_load_bytes": 18328776192,
+  "cuda_forward_ms": 2865.6015625,
+  "cuda_peak_bytes": 46381307904,
+  "effective_runtime_modes": {
+    "native_packed_matmul": 300
+  },
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "h3_dtype_views": 50,
+  "load_dtype": "default",
+  "load_seconds": 0.24018437741324306,
+  "model_dtype": "torch.bfloat16",
+  "move_to_cuda_seconds": 1.1501922300085425,
+  "num_blocks": 50,
+  "orbitquant_linear_count": 300,
+  "orbitquant_linear_cuda_ms": 1077.5865263938904,
+  "orbitquant_linear_inputs": {
+    "transformer_blocks.0.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    }
+  },
+  "orbitquant_shape_cuda_ms": {
+    "14336x5376": {
+      "calls": 50,
+      "total_ms": 230.40703773498535
+    },
+    "5376x28672": {
+      "calls": 50,
+      "total_ms": 354.5018243789673
+    },
+    "5376x7168": {
+      "calls": 150,
+      "total_ms": 368.4010241031647
+    },
+    "7168x5376": {
+      "calls": 50,
+      "total_ms": 124.27664017677307
+    }
+  },
+  "orbitquant_shapes": {
+    "14336x5376": 50,
+    "5376x28672": 50,
+    "5376x7168": 150,
+    "7168x5376": 50
+  },
+  "output_finite": [
+    true,
+    true
+  ],
+  "output_shapes": [
+    [
+      1,
+      10545,
+      96
+    ],
+    [
+      1,
+      207,
+      32
+    ]
+  ],
+  "runtime_mode": "native_packed_matmul",
+  "slowest_orbitquant_linears": [
+    {
+      "cuda_ms": 18.62041664123535,
+      "name": "transformer_blocks.0.attn.to_q",
+      "shape": "5376x7168"
+    },
+    {
+      "cuda_ms": 9.111200332641602,
+      "name": "transformer_blocks.0.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 8.695232391357422,
+      "name": "transformer_blocks.0.ff.net.2",
+      "shape": "14336x5376"
+    },
+    {
+      "cuda_ms": 7.266304016113281,
+      "name": "transformer_blocks.0.attn.to_out.0",
+      "shape": "7168x5376"
+    },
+    {
+      "cuda_ms": 7.081600189208984,
+      "name": "transformer_blocks.6.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.080512046813965,
+      "name": "transformer_blocks.27.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.074463844299316,
+      "name": "transformer_blocks.25.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.072256088256836,
+      "name": "transformer_blocks.10.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0702080726623535,
+      "name": "transformer_blocks.9.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0693440437316895,
+      "name": "transformer_blocks.4.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.068672180175781,
+      "name": "transformer_blocks.2.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.068160057067871,
+      "name": "transformer_blocks.11.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.067647933959961,
+      "name": "transformer_blocks.37.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.063136100769043,
+      "name": "transformer_blocks.24.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.063039779663086,
+      "name": "transformer_blocks.29.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.062047958374023,
+      "name": "transformer_blocks.32.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.062016010284424,
+      "name": "transformer_blocks.20.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.061503887176514,
+      "name": "transformer_blocks.15.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.061503887176514,
+      "name": "transformer_blocks.16.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.061024188995361,
+      "name": "transformer_blocks.22.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.059103965759277,
+      "name": "transformer_blocks.23.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.058047771453857,
+      "name": "transformer_blocks.47.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.056896209716797,
+      "name": "transformer_blocks.33.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.055935859680176,
+      "name": "transformer_blocks.40.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.054336071014404,
+      "name": "transformer_blocks.17.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.053952217102051,
+      "name": "transformer_blocks.39.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.053855895996094,
+      "name": "transformer_blocks.28.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.053311824798584,
+      "name": "transformer_blocks.3.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0502400398254395,
+      "name": "transformer_blocks.1.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.049215793609619,
+      "name": "transformer_blocks.5.ff.net.0.proj",
+      "shape": "5376x28672"
+    }
+  ],
+  "status": "pass",
+  "text_tokens": 512,
+  "torch": "2.8.0+cu128",
+  "transformer": "/workspace/orbitquant-h3-perf/model/transformer",
+  "transformer_block_cuda_ms": [
+    {
+      "cuda_ms": 310.5533752441406,
+      "name": "transformer_blocks.0"
+    },
+    {
+      "cuda_ms": 48.46879959106445,
+      "name": "transformer_blocks.1"
+    },
+    {
+      "cuda_ms": 48.629886627197266,
+      "name": "transformer_blocks.2"
+    },
+    {
+      "cuda_ms": 48.53862380981445,
+      "name": "transformer_blocks.3"
+    },
+    {
+      "cuda_ms": 48.653438568115234,
+      "name": "transformer_blocks.4"
+    },
+    {
+      "cuda_ms": 48.58992004394531,
+      "name": "transformer_blocks.5"
+    },
+    {
+      "cuda_ms": 48.66368103027344,
+      "name": "transformer_blocks.6"
+    },
+    {
+      "cuda_ms": 48.59689712524414,
+      "name": "transformer_blocks.7"
+    },
+    {
+      "cuda_ms": 48.5167350769043,
+      "name": "transformer_blocks.8"
+    },
+    {
+      "cuda_ms": 48.62108612060547,
+      "name": "transformer_blocks.9"
+    },
+    {
+      "cuda_ms": 48.51871871948242,
+      "name": "transformer_blocks.10"
+    },
+    {
+      "cuda_ms": 48.5648307800293,
+      "name": "transformer_blocks.11"
+    },
+    {
+      "cuda_ms": 48.545310974121094,
+      "name": "transformer_blocks.12"
+    },
+    {
+      "cuda_ms": 48.61705780029297,
+      "name": "transformer_blocks.13"
+    },
+    {
+      "cuda_ms": 48.762943267822266,
+      "name": "transformer_blocks.14"
+    },
+    {
+      "cuda_ms": 48.692222595214844,
+      "name": "transformer_blocks.15"
+    },
+    {
+      "cuda_ms": 48.60326385498047,
+      "name": "transformer_blocks.16"
+    },
+    {
+      "cuda_ms": 48.44297790527344,
+      "name": "transformer_blocks.17"
+    },
+    {
+      "cuda_ms": 48.3702392578125,
+      "name": "transformer_blocks.18"
+    },
+    {
+      "cuda_ms": 48.5167350769043,
+      "name": "transformer_blocks.19"
+    },
+    {
+      "cuda_ms": 48.67536163330078,
+      "name": "transformer_blocks.20"
+    },
+    {
+      "cuda_ms": 48.69500732421875,
+      "name": "transformer_blocks.21"
+    },
+    {
+      "cuda_ms": 48.48076629638672,
+      "name": "transformer_blocks.22"
+    },
+    {
+      "cuda_ms": 48.34979248046875,
+      "name": "transformer_blocks.23"
+    },
+    {
+      "cuda_ms": 48.50944137573242,
+      "name": "transformer_blocks.24"
+    },
+    {
+      "cuda_ms": 48.56480026245117,
+      "name": "transformer_blocks.25"
+    },
+    {
+      "cuda_ms": 48.68355178833008,
+      "name": "transformer_blocks.26"
+    },
+    {
+      "cuda_ms": 48.3375358581543,
+      "name": "transformer_blocks.27"
+    },
+    {
+      "cuda_ms": 48.75628662109375,
+      "name": "transformer_blocks.28"
+    },
+    {
+      "cuda_ms": 48.42652893066406,
+      "name": "transformer_blocks.29"
+    },
+    {
+      "cuda_ms": 48.335487365722656,
+      "name": "transformer_blocks.30"
+    },
+    {
+      "cuda_ms": 48.421470642089844,
+      "name": "transformer_blocks.31"
+    },
+    {
+      "cuda_ms": 48.22582244873047,
+      "name": "transformer_blocks.32"
+    },
+    {
+      "cuda_ms": 48.533119201660156,
+      "name": "transformer_blocks.33"
+    },
+    {
+      "cuda_ms": 48.40399932861328,
+      "name": "transformer_blocks.34"
+    },
+    {
+      "cuda_ms": 48.340030670166016,
+      "name": "transformer_blocks.35"
+    },
+    {
+      "cuda_ms": 48.41379165649414,
+      "name": "transformer_blocks.36"
+    },
+    {
+      "cuda_ms": 48.17622375488281,
+      "name": "transformer_blocks.37"
+    },
+    {
+      "cuda_ms": 48.51875305175781,
+      "name": "transformer_blocks.38"
+    },
+    {
+      "cuda_ms": 48.15116882324219,
+      "name": "transformer_blocks.39"
+    },
+    {
+      "cuda_ms": 48.514591217041016,
+      "name": "transformer_blocks.40"
+    },
+    {
+      "cuda_ms": 48.00166320800781,
+      "name": "transformer_blocks.41"
+    },
+    {
+      "cuda_ms": 48.32204818725586,
+      "name": "transformer_blocks.42"
+    },
+    {
+      "cuda_ms": 48.416385650634766,
+      "name": "transformer_blocks.43"
+    },
+    {
+      "cuda_ms": 47.402591705322266,
+      "name": "transformer_blocks.44"
+    },
+    {
+      "cuda_ms": 48.48284912109375,
+      "name": "transformer_blocks.45"
+    },
+    {
+      "cuda_ms": 48.11532974243164,
+      "name": "transformer_blocks.46"
+    },
+    {
+      "cuda_ms": 48.25551986694336,
+      "name": "transformer_blocks.47"
+    },
+    {
+      "cuda_ms": 48.40300750732422,
+      "name": "transformer_blocks.48"
+    },
+    {
+      "cuda_ms": 47.34415817260742,
+      "name": "transformer_blocks.49"
+    }
+  ],
+  "video_tokens": 10545,
+  "wall_seconds": 2.8667881772853434
+}

benchmarks/profiles/native-full-forward-transformer-ref-cold.json

@@ -0,0 +1,4629 @@
+{
+  "activation_backends": {
+    "triton_cuda_packed_w4": 300
+  },
+  "audio_tokens": 207,
+  "capability": [
+    12,
+    0
+  ],
+  "cuda": "12.8",
+  "cuda_after_load_bytes": 18328776192,
+  "cuda_forward_ms": 5536.7392578125,
+  "cuda_peak_bytes": 46381307904,
+  "effective_runtime_modes": {
+    "native_packed_matmul": 300
+  },
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "h3_dtype_views": 50,
+  "load_dtype": "default",
+  "load_seconds": 0.17170354118570685,
+  "model_dtype": "torch.bfloat16",
+  "move_to_cuda_seconds": 1.235624415334314,
+  "num_blocks": 50,
+  "orbitquant_linear_count": 300,
+  "orbitquant_linear_cuda_ms": 3368.3247916698456,
+  "orbitquant_linear_inputs": {
+    "transformer_blocks.0.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    }
+  },
+  "orbitquant_shape_cuda_ms": {
+    "14336x5376": {
+      "calls": 50,
+      "total_ms": 658.479362487793
+    },
+    "5376x28672": {
+      "calls": 50,
+      "total_ms": 493.38377046585083
+    },
+    "5376x7168": {
+      "calls": 150,
+      "total_ms": 1740.172526359558
+    },
+    "7168x5376": {
+      "calls": 50,
+      "total_ms": 476.2891323566437
+    }
+  },
+  "orbitquant_shapes": {
+    "14336x5376": 50,
+    "5376x28672": 50,
+    "5376x7168": 150,
+    "7168x5376": 50
+  },
+  "output_finite": [
+    true,
+    true
+  ],
+  "output_shapes": [
+    [
+      1,
+      10545,
+      96
+    ],
+    [
+      1,
+      207,
+      32
+    ]
+  ],
+  "runtime_mode": "native_packed_matmul",
+  "slowest_orbitquant_linears": [
+    {
+      "cuda_ms": 1387.7381591796875,
+      "name": "transformer_blocks.0.attn.to_q",
+      "shape": "5376x7168"
+    },
+    {
+      "cuda_ms": 439.46435546875,
+      "name": "transformer_blocks.0.ff.net.2",
+      "shape": "14336x5376"
+    },
+    {
+      "cuda_ms": 359.4147644042969,
+      "name": "transformer_blocks.0.attn.to_out.0",
+      "shape": "7168x5376"
+    },
+    {
+      "cuda_ms": 152.1229705810547,
+      "name": "transformer_blocks.0.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.239264011383057,
+      "name": "transformer_blocks.12.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.144959926605225,
+      "name": "transformer_blocks.9.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.108128070831299,
+      "name": "transformer_blocks.19.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.105184078216553,
+      "name": "transformer_blocks.7.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0979838371276855,
+      "name": "transformer_blocks.4.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0952959060668945,
+      "name": "transformer_blocks.6.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.088128089904785,
+      "name": "transformer_blocks.14.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.078400135040283,
+      "name": "transformer_blocks.20.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.073760032653809,
+      "name": "transformer_blocks.15.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.069248199462891,
+      "name": "transformer_blocks.13.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.067359924316406,
+      "name": "transformer_blocks.27.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.066656112670898,
+      "name": "transformer_blocks.1.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.066112041473389,
+      "name": "transformer_blocks.46.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.065248012542725,
+      "name": "transformer_blocks.34.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.061535835266113,
+      "name": "transformer_blocks.2.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0609917640686035,
+      "name": "transformer_blocks.10.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.057951927185059,
+      "name": "transformer_blocks.32.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.054912090301514,
+      "name": "transformer_blocks.44.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.054336071014404,
+      "name": "transformer_blocks.36.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.053855895996094,
+      "name": "transformer_blocks.39.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.053311824798584,
+      "name": "transformer_blocks.3.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.05299186706543,
+      "name": "transformer_blocks.24.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.052864074707031,
+      "name": "transformer_blocks.29.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0502400398254395,
+      "name": "transformer_blocks.22.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.048736095428467,
+      "name": "transformer_blocks.41.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.04307222366333,
+      "name": "transformer_blocks.17.ff.net.0.proj",
+      "shape": "5376x28672"
+    }
+  ],
+  "status": "pass",
+  "text_tokens": 512,
+  "torch": "2.8.0+cu128",
+  "transformer": "/workspace/orbitquant-h3-ref/model/transformer_ref",
+  "transformer_block_cuda_ms": [
+    {
+      "cuda_ms": 2941.07666015625,
+      "name": "transformer_blocks.0"
+    },
+    {
+      "cuda_ms": 48.6420783996582,
+      "name": "transformer_blocks.1"
+    },
+    {
+      "cuda_ms": 48.49164962768555,
+      "name": "transformer_blocks.2"
+    },
+    {
+      "cuda_ms": 48.473182678222656,
+      "name": "transformer_blocks.3"
+    },
+    {
+      "cuda_ms": 48.64112091064453,
+      "name": "transformer_blocks.4"
+    },
+    {
+      "cuda_ms": 48.53055953979492,
+      "name": "transformer_blocks.5"
+    },
+    {
+      "cuda_ms": 48.63804626464844,
+      "name": "transformer_blocks.6"
+    },
+    {
+      "cuda_ms": 48.6923828125,
+      "name": "transformer_blocks.7"
+    },
+    {
+      "cuda_ms": 48.50038528442383,
+      "name": "transformer_blocks.8"
+    },
+    {
+      "cuda_ms": 48.72953414916992,
+      "name": "transformer_blocks.9"
+    },
+    {
+      "cuda_ms": 48.53916931152344,
+      "name": "transformer_blocks.10"
+    },
+    {
+      "cuda_ms": 48.08662414550781,
+      "name": "transformer_blocks.11"
+    },
+    {
+      "cuda_ms": 48.662113189697266,
+      "name": "transformer_blocks.12"
+    },
+    {
+      "cuda_ms": 48.73068618774414,
+      "name": "transformer_blocks.13"
+    },
+    {
+      "cuda_ms": 48.63391876220703,
+      "name": "transformer_blocks.14"
+    },
+    {
+      "cuda_ms": 48.57455825805664,
+      "name": "transformer_blocks.15"
+    },
+    {
+      "cuda_ms": 48.3389778137207,
+      "name": "transformer_blocks.16"
+    },
+    {
+      "cuda_ms": 48.74409484863281,
+      "name": "transformer_blocks.17"
+    },
+    {
+      "cuda_ms": 48.08147048950195,
+      "name": "transformer_blocks.18"
+    },
+    {
+      "cuda_ms": 48.471649169921875,
+      "name": "transformer_blocks.19"
+    },
+    {
+      "cuda_ms": 48.72867202758789,
+      "name": "transformer_blocks.20"
+    },
+    {
+      "cuda_ms": 48.13267135620117,
+      "name": "transformer_blocks.21"
+    },
+    {
+      "cuda_ms": 48.483360290527344,
+      "name": "transformer_blocks.22"
+    },
+    {
+      "cuda_ms": 48.04719924926758,
+      "name": "transformer_blocks.23"
+    },
+    {
+      "cuda_ms": 48.55311965942383,
+      "name": "transformer_blocks.24"
+    },
+    {
+      "cuda_ms": 48.2775993347168,
+      "name": "transformer_blocks.25"
+    },
+    {
+      "cuda_ms": 47.991905212402344,
+      "name": "transformer_blocks.26"
+    },
+    {
+      "cuda_ms": 48.53667068481445,
+      "name": "transformer_blocks.27"
+    },
+    {
+      "cuda_ms": 48.02115249633789,
+      "name": "transformer_blocks.28"
+    },
+    {
+      "cuda_ms": 48.52803039550781,
+      "name": "transformer_blocks.29"
+    },
+    {
+      "cuda_ms": 48.025054931640625,
+      "name": "transformer_blocks.30"
+    },
+    {
+      "cuda_ms": 47.83679962158203,
+      "name": "transformer_blocks.31"
+    },
+    {
+      "cuda_ms": 48.23612976074219,
+      "name": "transformer_blocks.32"
+    },
+    {
+      "cuda_ms": 49.4167366027832,
+      "name": "transformer_blocks.33"
+    },
+    {
+      "cuda_ms": 48.58224105834961,
+      "name": "transformer_blocks.34"
+    },
+    {
+      "cuda_ms": 48.018463134765625,
+      "name": "transformer_blocks.35"
+    },
+    {
+      "cuda_ms": 47.99913787841797,
+      "name": "transformer_blocks.36"
+    },
+    {
+      "cuda_ms": 48.30006408691406,
+      "name": "transformer_blocks.37"
+    },
+    {
+      "cuda_ms": 47.80400085449219,
+      "name": "transformer_blocks.38"
+    },
+    {
+      "cuda_ms": 48.554622650146484,
+      "name": "transformer_blocks.39"
+    },
+    {
+      "cuda_ms": 48.00057601928711,
+      "name": "transformer_blocks.40"
+    },
+    {
+      "cuda_ms": 48.38880157470703,
+      "name": "transformer_blocks.41"
+    },
+    {
+      "cuda_ms": 47.99033737182617,
+      "name": "transformer_blocks.42"
+    },
+    {
+      "cuda_ms": 47.718048095703125,
+      "name": "transformer_blocks.43"
+    },
+    {
+      "cuda_ms": 48.561824798583984,
+      "name": "transformer_blocks.44"
+    },
+    {
+      "cuda_ms": 48.02803039550781,
+      "name": "transformer_blocks.45"
+    },
+    {
+      "cuda_ms": 48.63555145263672,
+      "name": "transformer_blocks.46"
+    },
+    {
+      "cuda_ms": 47.13932800292969,
+      "name": "transformer_blocks.47"
+    },
+    {
+      "cuda_ms": 47.67804718017578,
+      "name": "transformer_blocks.48"
+    },
+    {
+      "cuda_ms": 48.166561126708984,
+      "name": "transformer_blocks.49"
+    }
+  ],
+  "video_tokens": 10545,
+  "wall_seconds": 5.5382678848691285
+}

benchmarks/profiles/native-full-forward-transformer-ref-hot.json

@@ -0,0 +1,4629 @@
+{
+  "activation_backends": {
+    "triton_cuda_packed_w4": 300
+  },
+  "audio_tokens": 207,
+  "capability": [
+    12,
+    0
+  ],
+  "cuda": "12.8",
+  "cuda_after_load_bytes": 18328776192,
+  "cuda_forward_ms": 2889.388916015625,
+  "cuda_peak_bytes": 46381307904,
+  "effective_runtime_modes": {
+    "native_packed_matmul": 300
+  },
+  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+  "h3_dtype_views": 50,
+  "load_dtype": "default",
+  "load_seconds": 0.09053317736834288,
+  "model_dtype": "torch.bfloat16",
+  "move_to_cuda_seconds": 1.2328650671988726,
+  "num_blocks": 50,
+  "orbitquant_linear_count": 300,
+  "orbitquant_linear_cuda_ms": 1062.045917749405,
+  "orbitquant_linear_inputs": {
+    "transformer_blocks.0.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.0.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.1.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.10.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.11.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.12.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.13.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.14.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.15.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.16.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.17.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.18.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.19.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.2.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.20.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.21.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.22.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.23.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.24.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.25.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.26.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.27.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.28.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.29.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.3.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.30.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.31.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.32.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.33.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.34.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.35.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.36.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.37.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.38.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.39.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.4.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.40.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.41.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.42.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.43.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.44.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.45.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.46.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.47.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.48.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.49.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.5.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.6.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.7.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.8.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_k": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_out.0": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        7168
+      ],
+      "stride": [
+        80740352,
+        7168,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_q": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.attn.to_v": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.0.proj": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        5376
+      ],
+      "stride": [
+        60555264,
+        5376,
+        1
+      ]
+    },
+    "transformer_blocks.9.ff.net.2": {
+      "contiguous": true,
+      "dtype": "torch.bfloat16",
+      "shape": [
+        1,
+        11264,
+        14336
+      ],
+      "stride": [
+        161480704,
+        14336,
+        1
+      ]
+    }
+  },
+  "orbitquant_shape_cuda_ms": {
+    "14336x5376": {
+      "calls": 50,
+      "total_ms": 225.03200006484985
+    },
+    "5376x28672": {
+      "calls": 50,
+      "total_ms": 347.8003191947937
+    },
+    "5376x7168": {
+      "calls": 150,
+      "total_ms": 365.89542269706726
+    },
+    "7168x5376": {
+      "calls": 50,
+      "total_ms": 123.31817579269409
+    }
+  },
+  "orbitquant_shapes": {
+    "14336x5376": 50,
+    "5376x28672": 50,
+    "5376x7168": 150,
+    "7168x5376": 50
+  },
+  "output_finite": [
+    true,
+    true
+  ],
+  "output_shapes": [
+    [
+      1,
+      10545,
+      96
+    ],
+    [
+      1,
+      207,
+      32
+    ]
+  ],
+  "runtime_mode": "native_packed_matmul",
+  "slowest_orbitquant_linears": [
+    {
+      "cuda_ms": 18.906143188476562,
+      "name": "transformer_blocks.0.attn.to_q",
+      "shape": "5376x7168"
+    },
+    {
+      "cuda_ms": 9.0982084274292,
+      "name": "transformer_blocks.0.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 8.691136360168457,
+      "name": "transformer_blocks.0.ff.net.2",
+      "shape": "14336x5376"
+    },
+    {
+      "cuda_ms": 7.211008071899414,
+      "name": "transformer_blocks.0.attn.to_out.0",
+      "shape": "7168x5376"
+    },
+    {
+      "cuda_ms": 7.1143999099731445,
+      "name": "transformer_blocks.6.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.098368167877197,
+      "name": "transformer_blocks.1.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0820159912109375,
+      "name": "transformer_blocks.3.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.071263790130615,
+      "name": "transformer_blocks.47.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0655999183654785,
+      "name": "transformer_blocks.2.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0641279220581055,
+      "name": "transformer_blocks.9.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.058080196380615,
+      "name": "transformer_blocks.24.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.055903911590576,
+      "name": "transformer_blocks.34.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.050911903381348,
+      "name": "transformer_blocks.4.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.049727916717529,
+      "name": "transformer_blocks.19.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.049215793609619,
+      "name": "transformer_blocks.5.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.046656131744385,
+      "name": "transformer_blocks.44.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.04585599899292,
+      "name": "transformer_blocks.7.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0456318855285645,
+      "name": "transformer_blocks.39.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.0415358543396,
+      "name": "transformer_blocks.12.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.039487838745117,
+      "name": "transformer_blocks.21.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.036448001861572,
+      "name": "transformer_blocks.29.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.036448001861572,
+      "name": "transformer_blocks.42.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.028736114501953,
+      "name": "transformer_blocks.26.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 7.02566385269165,
+      "name": "transformer_blocks.14.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 6.987775802612305,
+      "name": "transformer_blocks.37.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 6.983263969421387,
+      "name": "transformer_blocks.32.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 6.972415924072266,
+      "name": "transformer_blocks.17.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 6.934175968170166,
+      "name": "transformer_blocks.27.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 6.928383827209473,
+      "name": "transformer_blocks.22.ff.net.0.proj",
+      "shape": "5376x28672"
+    },
+    {
+      "cuda_ms": 6.850048065185547,
+      "name": "transformer_blocks.10.ff.net.0.proj",
+      "shape": "5376x28672"
+    }
+  ],
+  "status": "pass",
+  "text_tokens": 512,
+  "torch": "2.8.0+cu128",
+  "transformer": "/workspace/orbitquant-h3-ref/model/transformer_ref",
+  "transformer_block_cuda_ms": [
+    {
+      "cuda_ms": 311.4284362792969,
+      "name": "transformer_blocks.0"
+    },
+    {
+      "cuda_ms": 48.51638412475586,
+      "name": "transformer_blocks.1"
+    },
+    {
+      "cuda_ms": 48.574527740478516,
+      "name": "transformer_blocks.2"
+    },
+    {
+      "cuda_ms": 48.55510330200195,
+      "name": "transformer_blocks.3"
+    },
+    {
+      "cuda_ms": 48.56742477416992,
+      "name": "transformer_blocks.4"
+    },
+    {
+      "cuda_ms": 48.5162239074707,
+      "name": "transformer_blocks.5"
+    },
+    {
+      "cuda_ms": 48.63600158691406,
+      "name": "transformer_blocks.6"
+    },
+    {
+      "cuda_ms": 48.58127975463867,
+      "name": "transformer_blocks.7"
+    },
+    {
+      "cuda_ms": 48.1910400390625,
+      "name": "transformer_blocks.8"
+    },
+    {
+      "cuda_ms": 48.545345306396484,
+      "name": "transformer_blocks.9"
+    },
+    {
+      "cuda_ms": 48.06208038330078,
+      "name": "transformer_blocks.10"
+    },
+    {
+      "cuda_ms": 47.823455810546875,
+      "name": "transformer_blocks.11"
+    },
+    {
+      "cuda_ms": 48.4901123046875,
+      "name": "transformer_blocks.12"
+    },
+    {
+      "cuda_ms": 47.92486572265625,
+      "name": "transformer_blocks.13"
+    },
+    {
+      "cuda_ms": 48.53456115722656,
+      "name": "transformer_blocks.14"
+    },
+    {
+      "cuda_ms": 48.27961730957031,
+      "name": "transformer_blocks.15"
+    },
+    {
+      "cuda_ms": 47.93564987182617,
+      "name": "transformer_blocks.16"
+    },
+    {
+      "cuda_ms": 48.035457611083984,
+      "name": "transformer_blocks.17"
+    },
+    {
+      "cuda_ms": 47.76406478881836,
+      "name": "transformer_blocks.18"
+    },
+    {
+      "cuda_ms": 48.521854400634766,
+      "name": "transformer_blocks.19"
+    },
+    {
+      "cuda_ms": 48.037376403808594,
+      "name": "transformer_blocks.20"
+    },
+    {
+      "cuda_ms": 48.364158630371094,
+      "name": "transformer_blocks.21"
+    },
+    {
+      "cuda_ms": 48.13004684448242,
+      "name": "transformer_blocks.22"
+    },
+    {
+      "cuda_ms": 47.60380935668945,
+      "name": "transformer_blocks.23"
+    },
+    {
+      "cuda_ms": 48.526206970214844,
+      "name": "transformer_blocks.24"
+    },
+    {
+      "cuda_ms": 48.04182434082031,
+      "name": "transformer_blocks.25"
+    },
+    {
+      "cuda_ms": 47.844512939453125,
+      "name": "transformer_blocks.26"
+    },
+    {
+      "cuda_ms": 48.17929458618164,
+      "name": "transformer_blocks.27"
+    },
+    {
+      "cuda_ms": 47.58198547363281,
+      "name": "transformer_blocks.28"
+    },
+    {
+      "cuda_ms": 48.438785552978516,
+      "name": "transformer_blocks.29"
+    },
+    {
+      "cuda_ms": 48.02822494506836,
+      "name": "transformer_blocks.30"
+    },
+    {
+      "cuda_ms": 46.90595245361328,
+      "name": "transformer_blocks.31"
+    },
+    {
+      "cuda_ms": 48.04880142211914,
+      "name": "transformer_blocks.32"
+    },
+    {
+      "cuda_ms": 47.76803207397461,
+      "name": "transformer_blocks.33"
+    },
+    {
+      "cuda_ms": 48.554656982421875,
+      "name": "transformer_blocks.34"
+    },
+    {
+      "cuda_ms": 47.988670349121094,
+      "name": "transformer_blocks.35"
+    },
+    {
+      "cuda_ms": 47.18297576904297,
+      "name": "transformer_blocks.36"
+    },
+    {
+      "cuda_ms": 48.055423736572266,
+      "name": "transformer_blocks.37"
+    },
+    {
+      "cuda_ms": 47.75484848022461,
+      "name": "transformer_blocks.38"
+    },
+    {
+      "cuda_ms": 48.46246337890625,
+      "name": "transformer_blocks.39"
+    },
+    {
+      "cuda_ms": 47.13225555419922,
+      "name": "transformer_blocks.40"
+    },
+    {
+      "cuda_ms": 47.67193603515625,
+      "name": "transformer_blocks.41"
+    },
+    {
+      "cuda_ms": 47.967838287353516,
+      "name": "transformer_blocks.42"
+    },
+    {
+      "cuda_ms": 47.16921615600586,
+      "name": "transformer_blocks.43"
+    },
+    {
+      "cuda_ms": 47.7097282409668,
+      "name": "transformer_blocks.44"
+    },
+    {
+      "cuda_ms": 47.17932891845703,
+      "name": "transformer_blocks.45"
+    },
+    {
+      "cuda_ms": 47.825439453125,
+      "name": "transformer_blocks.46"
+    },
+    {
+      "cuda_ms": 48.40099334716797,
+      "name": "transformer_blocks.47"
+    },
+    {
+      "cuda_ms": 47.14352035522461,
+      "name": "transformer_blocks.48"
+    },
+    {
+      "cuda_ms": 46.874176025390625,
+      "name": "transformer_blocks.49"
+    }
+  ],
+  "video_tokens": 10545,
+  "wall_seconds": 2.890391333028674
+}

comfyui/examples/h264/comfyui-ref2va-608x480.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02233da18b3c01761889df2206181e0fc7b9f120165b37a0fe4c9f7bbdda0096
+size 581273

comfyui/examples/h264/comfyui-t2va-608x480.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cceecedd28dfc593cf5e61145836405d18475bd8109b2ef7a984d96d91a13e6
+size 611905

comfyui/examples/hevc/comfyui-ref2va-608x480.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c3040f81aefdb14b30047a5c972347d012d72dbfa026b8a9b2e245e555b7996
+size 470007

comfyui/examples/hevc/comfyui-t2va-608x480.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc31c456f9fdfae29c4e89d063baa953bbd73141c05a237ed43006336661eaab
+size 423393

comfyui/report.json

@@ -0,0 +1,96 @@
+{
+  "status": "pass",
+  "date": "2026-08-04",
+  "hardware": {
+    "provider": "RunPod",
+    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
+    "gpu_memory_mib": 97887,
+    "template": "runpod-torch-v280",
+    "image": "runpod/pytorch:1.0.2-cu1281-torch280-ubuntu2404"
+  },
+  "revisions": {
+    "comfyui": "9a9fdb10ed144ce760d9682cb247526ea23cc525",
+    "comfyui_frontend": "1.47.12",
+    "comfyui_orbitquant_candidate": "51d5473bc2bd4f14572724d362770c8ffacb10ed",
+    "model_release_tested": "e434bbea523349576e7c3d2f6090744aa4597123",
+    "source_model": "73372e6cf53e414edd3ab03e357717fb0602e758",
+    "orbitquant": "0.9.1",
+    "diffusers": "abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc",
+    "official_workflow_templates": "7653f1cdef1d92394b6ef9946018c0a8aa4136b8"
+  },
+  "node_surface": {
+    "release_loader": "OrbitQuantReleaseLoader",
+    "video_generator": "OrbitQuantGenerateVideo",
+    "release_type": "ORBITQUANT_RELEASE",
+    "output_type": "VIDEO",
+    "model_specific_public_nodes": false,
+    "adapter": "minimax_h3"
+  },
+  "workflow": {
+    "source": "Comfy-Org/workflow_templates/templates/video_minimax_h3_t2v.json",
+    "ui_file": "workflows/minimax-h3-t2va-ui.json",
+    "api_file_from_live_graph": "workflows/minimax-h3-public-ui-api.json",
+    "workflow_image_export": "workflow-export.png",
+    "live_import": {
+      "node_count": 6,
+      "missing_nodes": [],
+      "links": 2,
+      "seed": 42,
+      "width": 608,
+      "height": 480,
+      "num_frames": 124,
+      "steps": 50
+    },
+    "terminal_prompt_id": "823b3407-ae85-43fd-b51a-129ea7a11f4d",
+    "save_video_output": {
+      "codec": "h264",
+      "width": 608,
+      "height": 480,
+      "frames": 124,
+      "fps": 24,
+      "audio": "aac stereo 32000 Hz",
+      "duration_seconds": 5.175
+    }
+  },
+  "runs": {
+    "t2va": {
+      "status": "pass",
+      "prompt_id": "32d20b0b-f8b9-4a8f-aa25-98db4ccc9644",
+      "component": "transformer",
+      "generation_seconds": 137.13920322200283,
+      "generation_peak_vram_gib": 18.90608207695186,
+      "decode_seconds": 5.4824893129989505,
+      "decode_peak_vram_gib": 11.286694642156363,
+      "native_w4_modules": "300/300",
+      "offload": "manual_stage_offload",
+      "text_encoder_after_generation": "cpu",
+      "vae_dtype": "fp32",
+      "audio_vae_dtype": "fp32",
+      "semantic_review": "pass: stable face, eyes, lips, wet-hair and jacket texture; no grid, ghosting, or temporal collapse"
+    },
+    "ref2va": {
+      "status": "pass",
+      "prompt_id": "4aa7325a-98e2-44ea-9f24-46ec404213b1",
+      "component": "transformer_ref",
+      "generation_seconds": 318.855212665163,
+      "generation_peak_vram_gib": 30.459273928776383,
+      "decode_seconds": 5.50284016598016,
+      "decode_peak_vram_gib": 11.286694642156363,
+      "native_w4_modules": "300/300",
+      "offload": "components_manager_auto_cpu_offload",
+      "offload_reserve_margin": "64GB",
+      "text_encoder_after_generation": "cpu",
+      "vae_dtype": "fp32",
+      "audio_vae_dtype": "fp32",
+      "semantic_review": "pass: face geometry, eyes, lips, visor reflection and suit texture remain coherent through the push-in"
+    }
+  },
+  "media": {
+    "h264_t2va_sha256": "4cceecedd28dfc593cf5e61145836405d18475bd8109b2ef7a984d96d91a13e6",
+    "hevc_t2va_sha256": "bc31c456f9fdfae29c4e89d063baa953bbd73141c05a237ed43006336661eaab",
+    "h264_ref2va_sha256": "02233da18b3c01761889df2206181e0fc7b9f120165b37a0fe4c9f7bbdda0096",
+    "hevc_ref2va_sha256": "0c3040f81aefdb14b30047a5c972347d012d72dbfa026b8a9b2e245e555b7996",
+    "workflow_sha256": "7c1a82427b80203e4252726a0ab3a336a6859fe2ff43302a07de00bce5722acb",
+    "workflow_image_export_sha256": "df49d477b234e71c23e0a7d457d66315408333db62433885db4b20f3606328f0"
+  }
+}

comfyui/workflows/MiniMax-H3-OrbitQuant-Ref2VA-api.json

@@ -0,0 +1,32 @@
+{
+  "1": {
+    "class_type": "OrbitQuantReleaseLoader",
+    "inputs": {
+      "model_path": "/workspace/models/MiniMax-H3-OrbitQuant-W4A4"
+    },
+    "_meta": {
+      "title": "OrbitQuant Release Loader"
+    }
+  },
+  "2": {
+    "class_type": "OrbitQuantGenerateVideo",
+    "inputs": {
+      "release": [
+        "1",
+        0
+      ],
+      "prompt": "Single continuous cinematic shot of the referenced astronaut. The astronaut turns toward the camera and raises one gloved hand as the camera makes a fast smooth push-in toward the helmet visor. The face, eyes, visor reflections, suit patches, and fabric texture remain coherent during motion. No cuts.",
+      "task": "ref2va",
+      "reference_path": "/workspace/evidence/astronaut.png",
+      "seed": 314159,
+      "width": 608,
+      "height": 480,
+      "num_frames": 124,
+      "steps": 50,
+      "filename_prefix": "orbitquant/minimax-h3-ref2va"
+    },
+    "_meta": {
+      "title": "OrbitQuant Generate Video"
+    }
+  }
+}

comfyui/workflows/MiniMax-H3-OrbitQuant-T2VA-api.json

@@ -0,0 +1,47 @@
+{
+  "92": {
+    "inputs": {
+      "filename_prefix": "video/MiniMax_H3_OrbitQuant",
+      "format": "auto",
+      "codec": "auto",
+      "video": [
+        "105",
+        0
+      ]
+    },
+    "class_type": "SaveVideo",
+    "_meta": {
+      "title": "Save Video"
+    }
+  },
+  "105": {
+    "inputs": {
+      "prompt": "Single continuous live-action shot of a young woman running toward the camera in rain. The camera rapidly dollies backward, then smoothly pushes into a tight close-up while she turns her head and says in Russian: «Я здесь». Stable face identity, natural eyes and lips, coherent wet hair and jacket texture, strong motion, no cuts.",
+      "task": "t2va",
+      "reference_path": "",
+      "seed": 42,
+      "width": 608,
+      "height": 480,
+      "num_frames": 124,
+      "steps": 50,
+      "filename_prefix": "orbitquant/minimax-h3-t2va",
+      "release": [
+        "119",
+        0
+      ]
+    },
+    "class_type": "OrbitQuantGenerateVideo",
+    "_meta": {
+      "title": "Generate Video (Verified 608 × 480)"
+    }
+  },
+  "119": {
+    "inputs": {
+      "model_path": "/workspace/models/MiniMax-H3-OrbitQuant-W4A4"
+    },
+    "class_type": "OrbitQuantReleaseLoader",
+    "_meta": {
+      "title": "OrbitQuant Release Loader (Model)"
+    }
+  }
+}

comfyui/workflows/MiniMax-H3-OrbitQuant-T2VA.json

@@ -0,0 +1,139 @@
+{
+  "id": "d751242b-9f76-4c66-9183-449980185e86",
+  "revision": 0,
+  "last_node_id": 119,
+  "last_link_id": 2,
+  "nodes": [
+    {
+      "id": 117,
+      "type": "Note",
+      "pos": [-2550, 4840],
+      "size": [440, 1080],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Note: Model Links",
+      "properties": {},
+      "widgets_values": ["MODEL AND NODE LINKS\n\nModel release:\nhuggingface.co/WaveCut/\nMiniMax-H3-OrbitQuant-W4A4\n\nNode pack:\ngithub.com/iamwavecut/\nComfyUI-OrbitQuant\n\nOriginal model:\nhuggingface.co/MiniMaxAI/MiniMax-H3\n\nMODEL DIRECTORY\n\nmodels/MiniMax-H3-OrbitQuant-W4A4/\n  transformer/       W4A4\n  transformer_ref/   W4A4\n  text_encoder/      W4A4\n  vae/               source FP32\n  audio_vae/         source FP32\n  scripts/\n  quantization_manifest.json\n  comfyui_orbitquant.json\n\nSet this directory in\nOrbitQuant Release Loader.\n\nRelease configuration selects an\nallowlisted internal adapter. Public node\nclasses stay model-agnostic.\n\nRUNTIME\n\nUse current ComfyUI, OrbitQuant 0.9.x,\nand the H3 Diffusers revision pinned\nin the model card."],
+      "color": "#222",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 116,
+      "type": "Note",
+      "pos": [-2080, 4840],
+      "size": [450, 1080],
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Note: MiniMax H3 OrbitQuant",
+      "properties": {},
+      "widgets_values": ["MINIMAX H3 — ORBITQUANT W4A4\n\nThis workflow follows ComfyUI's bundled\nMiniMax H3 Text to Video preset.\n\nLayout:\n- model and usage notes on the left\n- compact generation in the center\n- core Save Video on the right\n\nOfficial preset:\nComfy-Org/workflow_templates\nvideo_minimax_h3_t2v.json\n\nKEY INPUTS\n\nprompt\nDescribe the shot, camera motion, dialogue,\nsound effects, and music in one block.\n\ntask\nt2va for text-to-video. Use ref2va plus a\nlocal reference_path for reference-to-video.\n\nwidth / height\nVerified at 608 × 480.\n\nnum_frames\n124 frames: the shortest verified\nfive-second H3 packing sequence.\n\nsteps\n50 sigma points / 49 denoiser forwards.\n\nOFFLOAD AND PRECISION\n\nThe text encoder enters GPU for conditioning\nand returns to CPU afterward.\n\nT2VA uses manual stage offload. Ref2VA uses\nH3 ComponentsManager for source FP32 VAE\nreference encoding. Visual and audio decode\nuse untouched source FP32 VAEs."],
+      "color": "#222",
+      "bgcolor": "#000"
+    },
+    {
+      "id": 119,
+      "type": "OrbitQuantReleaseLoader",
+      "pos": [-1580, 4840],
+      "size": [450, 150],
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [
+        {"name": "release", "type": "ORBITQUANT_RELEASE", "links": [1], "slot_index": 0},
+        {"name": "summary_json", "type": "STRING", "links": null, "slot_index": 1}
+      ],
+      "title": "OrbitQuant Release Loader (Model)",
+      "properties": {"Node name for S&R": "OrbitQuantReleaseLoader"},
+      "widgets_values": ["/workspace/models/MiniMax-H3-OrbitQuant-W4A4"]
+    },
+    {
+      "id": 105,
+      "type": "OrbitQuantGenerateVideo",
+      "pos": [-1080, 4840],
+      "size": [500, 630],
+      "flags": {},
+      "order": 3,
+      "mode": 0,
+      "inputs": [
+        {"name": "release", "type": "ORBITQUANT_RELEASE", "link": 1},
+        {"name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
+        {"name": "task", "type": "COMBO", "widget": {"name": "task"}, "link": null},
+        {"name": "reference_path", "type": "STRING", "widget": {"name": "reference_path"}, "link": null},
+        {"name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null},
+        {"name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
+        {"name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
+        {"name": "num_frames", "type": "INT", "widget": {"name": "num_frames"}, "link": null},
+        {"name": "steps", "type": "INT", "widget": {"name": "steps"}, "link": null},
+        {"name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null}
+      ],
+      "outputs": [
+        {"name": "video", "type": "VIDEO", "links": [2], "slot_index": 0},
+        {"name": "report_json", "type": "STRING", "links": null, "slot_index": 1}
+      ],
+      "title": "Generate Video (Verified 608 × 480)",
+      "properties": {"Node name for S&R": "OrbitQuantGenerateVideo", "previewExposures": []},
+      "widgets_values": [
+        "Single continuous live-action shot of a young woman running toward the camera in rain. The camera rapidly dollies backward, then smoothly pushes into a tight close-up while she turns her head and says in Russian: «Я здесь». Stable face identity, natural eyes and lips, coherent wet hair and jacket texture, strong motion, no cuts.",
+        "t2va",
+        "",
+        42,
+        608,
+        480,
+        124,
+        50,
+        "orbitquant/minimax-h3-t2va"
+      ]
+    },
+    {
+      "id": 92,
+      "type": "SaveVideo",
+      "pos": [-540, 4840],
+      "size": [850, 126],
+      "flags": {},
+      "order": 4,
+      "mode": 0,
+      "inputs": [
+        {"name": "video", "type": "VIDEO", "link": 2}
+      ],
+      "outputs": [
+        {"name": "video", "type": "VIDEO", "links": null}
+      ],
+      "properties": {},
+      "widgets_values": ["video/MiniMax_H3_OrbitQuant", "auto", "auto"]
+    },
+    {
+      "id": 118,
+      "type": "Note",
+      "pos": [-1580, 5040],
+      "size": [450, 520],
+      "flags": {},
+      "order": 5,
+      "mode": 0,
+      "inputs": [],
+      "outputs": [],
+      "title": "Note: Verified Profile",
+      "properties": {},
+      "widgets_values": ["VERIFIED 480P PROFILE\n\nwidth                 608\nheight                480\nframes                124\nfps                    24\nsigma points           50\ndenoiser forwards      49\nvisual VAE     source FP32\naudio VAE      source FP32\n\nH3 accepts 5–15 seconds. The runner rejects\nshorter frame counts before allocating the\ndenoiser, preventing expensive invalid jobs.\n\nScheduler checkpoints, the latent bundle,\nmetrics, and stage logs are persisted as soon\nas each artifact exists."],
+      "color": "#222",
+      "bgcolor": "#000"
+    }
+  ],
+  "links": [
+    [1, 119, 0, 105, 0, "ORBITQUANT_RELEASE"],
+    [2, 105, 0, 92, 0, "VIDEO"]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "frontendVersion": "1.47.12",
+    "ds": {"scale": 0.58, "offset": [1550, -2550]}
+  },
+  "version": 0.4
+}

comfyui_orbitquant.json

@@ -0,0 +1,15 @@
+{
+  "schema_version": 1,
+  "adapter": "minimax_h3",
+  "media_type": "video",
+  "tasks": [
+    "t2va",
+    "ref2va"
+  ],
+  "defaults": {
+    "width": 608,
+    "height": 480,
+    "num_frames": 124,
+    "steps": 50
+  }
+}

docs/QA-about-License.md

@@ -0,0 +1,65 @@
+# Q&A About License
+
+## Why is MiniMax-H3's open-weight license currently limited to the EU, UK, South Korea, and US?
+
+MiniMax-H3 was built with the goal of global availability. The current territory scope is not about excluding specific countries or regions, but about recognizing that video generation models are facing a more complex and rapidly evolving regulatory environment compared with text or code models.
+
+Regions such as the EU, UK, South Korea, and the US are currently developing or enforcing AI-related regulations that may have specific implications for generative video models, especially around areas such as likeness generation, copyright, content safety, and responsible deployment.
+
+- The EU AI Act has started enforcement, while practical requirements for models capable of generating video and likeness-related content are still evolving.
+- Similar regulatory uncertainties exist in the UK and South Korea regarding AI-generated content and video generation.
+- In the US, AI regulation remains a rapidly changing landscape, and MiniMax is also involved in ongoing copyright-related legal proceedings specifically concerning generative video AI.
+
+For open-weight models, once the weights are released, developers can deploy and modify them independently. This creates different compliance challenges compared with hosted services.
+
+We had two options:
+
+1. Wait until every jurisdiction reaches complete regulatory clarity before releasing open weights, which could take a long time while AI technology continues to evolve.
+2. Release the model now with a transparent license scope, while continuing to evaluate and expand availability.
+
+We chose the second approach. The current limitation means "not yet", not "not ever."
+
+## Why can MiniMax-H3 API be used globally if open weights are restricted in some regions?
+
+The difference is the distribution model.
+
+The main concern is not the existence of MiniMax-H3 itself, but the ability to control compliance after open weights leave our infrastructure.
+
+For API access, MiniMax operates the serving infrastructure and can enforce appropriate safeguards, including:
+
+- Protection against misuse involving minors.
+- Copyright-related compliance measures.
+- Content safety controls.
+- Compliance with applicable laws and regulations.
+
+With open weights, users can independently deploy, modify, and distribute the model, which makes it much harder to ensure the same level of compliance.
+
+Therefore, the API and open-weight release follow different approaches:
+
+- API: Globally available with built-in safeguards and responsible-use controls.
+- Open weights: Temporarily limited in certain regions until the regulatory and compliance framework becomes clearer.
+
+## Can organizations in restricted regions still use MiniMax-H3?
+
+Yes.
+
+Organizations in these regions can apply for a formal license. After reviewing the deployment scenario and confirming that appropriate compliance controls and safeguards are implemented, MiniMax may authorize usage.
+
+[Application form](https://vrfi1sk8a0.feishu.cn/share/base/form/shrcnD9XM1zYI9VFJxTEbt0d19g)
+
+Through authorized deployments, MiniMax can ensure that MiniMax-H3 is used responsibly while meeting local legal and regulatory requirements.
+
+## Will MiniMax expand open-weight availability in the future?
+
+Yes. We will continue monitoring legal developments and reassessing the territory scope.
+
+Our commitments:
+
+- Continuously review regulations and compliance requirements across regions.
+- Keep MiniMax-H3 API available globally, so users can continue accessing the model.
+- Clearly communicate any future license changes instead of making silent updates.
+- Listen to feedback from developers, researchers, and organizations affected by these restrictions.
+
+Our goal remains the same: bringing intelligence to everyone with responsible AI use.
+
+The current license scope reflects today's regulatory reality, not our long-term vision.

examples/h264/source-bf16-control.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87044e59a72a2b8ed3df285de044a097d7366f3004dcfc26ffd11fc71ed32d17
+size 635349

examples/h264/w4-text-control.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:513f09b20cb4e59a47a2f4ff19e544cb29d41870ee2c4759ad0a2ff6cb16fe0c
+size 753550

examples/h264/w4a4-native-t2va.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:679c0b690fce47ed2dd19ba7fa185d0cea69ee3a3b308a84faa909ce0762d24b
+size 815089

examples/hevc/source-bf16-control.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb4eb5d19d231887c6dfba5be70ac3d9107fe076a5f7c69e6fa77ecc0ad5ced9
+size 375788

examples/hevc/w4-text-control.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea89a3cc2a37c4294da4142f22d9d3778f1d086efb6125d73e1d5ccf392c6cbb
+size 449021

examples/hevc/w4a4-native-t2va.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bef87dbecbd9963e6febb43fb44cc2caf142192e59f7deee13950e55789b235
+size 474343

examples/reviews/source-bf16-control.json

@@ -0,0 +1,40 @@
+{
+  "status": "pass",
+  "review_type": "manual_visual_temporal",
+  "reviewed_frame_indices": [
+    0,
+    5,
+    10,
+    15,
+    20,
+    25,
+    30,
+    35,
+    40,
+    45,
+    50,
+    55,
+    60,
+    65,
+    70,
+    75,
+    80,
+    85,
+    90,
+    95,
+    100,
+    105,
+    110,
+    115,
+    120,
+    123
+  ],
+  "prompt_subject_recognizable": true,
+  "coherent_motion": true,
+  "repeating_tile_artifacts": false,
+  "motion_ghosting": false,
+  "texture_breakup": false,
+  "face_integrity": true,
+  "full_resolution_frame_reviewed": true,
+  "notes": "Dense every-fifth-frame and 6 FPS tail review preserves one face, aligned eyes and lips, and persistent wet-skin texture. Fast running frames contain ordinary motion blur that resolves in adjacent frames. This review is visual only; generated audio still requires a listening gate."
+}

examples/reviews/w4-text-control.json

@@ -0,0 +1,42 @@
+{
+  "status": "pass",
+  "review_type": "manual_visual_temporal",
+  "reviewed_frame_indices": [
+    0,
+    5,
+    10,
+    15,
+    20,
+    25,
+    30,
+    35,
+    40,
+    45,
+    50,
+    55,
+    60,
+    65,
+    70,
+    75,
+    80,
+    85,
+    90,
+    95,
+    100,
+    105,
+    110,
+    115,
+    120,
+    123
+  ],
+  "prompt_subject_recognizable": true,
+  "coherent_motion": true,
+  "repeating_tile_artifacts": false,
+  "motion_ghosting": false,
+  "texture_breakup": false,
+  "face_integrity": true,
+  "full_resolution_frame_reviewed": true,
+  "audio_listening_status": "pass",
+  "audio_reviewed_by": "user",
+  "notes": "Dense every-fifth-frame and 6 FPS tail review preserves one face, aligned eyes and lips, and persistent wet-skin texture. Full-resolution frames 80 and 123 were inspected directly. The user reviewed the embedded video with audio and judged it ideal."
+}

examples/reviews/w4a4-native-t2va.json

@@ -0,0 +1,35 @@
+{
+  "status": "pass",
+  "review_type": "manual_visual_temporal_and_audio_signal",
+  "reviewed_frame_indices": [
+    0,
+    24,
+    48,
+    60,
+    72,
+    80,
+    84,
+    88,
+    90,
+    92,
+    96,
+    100,
+    104,
+    108,
+    110,
+    112,
+    116,
+    120,
+    123
+  ],
+  "prompt_subject_recognizable": true,
+  "coherent_motion": true,
+  "repeating_tile_artifacts": false,
+  "motion_ghosting": false,
+  "texture_breakup": false,
+  "face_integrity": true,
+  "full_resolution_frame_reviewed": true,
+  "audio_signal_integrity": true,
+  "audio_user_listening_status": "pending",
+  "notes": "Full-timeline and dense late-motion grids plus full-resolution frames 60, 90, 110, and 123 preserve one coherent face, aligned eyes, connected lip motion, teeth, rain, and skin texture without the earlier grid, doubled contour, ghosting, or breakup. The HEVC/AAC file decodes to 124 frames at 608x480. Audio has no NaN/Inf samples or clipping, peaks at -0.27 dBFS, and has no persistent high-frequency narrow-band tone; energy above 8 kHz is 1.42%, below the accepted W4A16 control's 2.99%."
+}