Requantize W2A3 with mixed-width low-bit protection

README.md

@@ -19,20 +19,9 @@ OrbitQuant is a calibration-free post-training quantization method for image and
 Install OrbitQuant and the Hugging Face runtime dependencies:
 
 ```bash
-pip install "orbitquant[hf,kernels]>=0.6.0"
+pip install "orbitquant[hf,kernels]>=0.4.0"
 ```
 
-The optimized native kernel package is provisioned automatically at first
-model load: OrbitQuant derives the runtime variant (torch minor and CUDA
-version for CUDA, the torch stable ABI for CPU, plus OS and architecture),
-downloads the matching prebuilt wheel from the
-[OrbitQuant kernels release](https://github.com/iamwavecut/OrbitQuant/releases/tag/kernels-v1)
-with checksum verification, and caches it under `~/.cache/orbitquant/kernels`.
-Without a matching variant, CUDA inference falls back to the Triton packed
-path. Run `orbitquant kernels-install` to provision explicitly. For ComfyUI,
-the [ComfyUI-OrbitQuant](https://github.com/iamwavecut/ComfyUI-OrbitQuant)
-node pack provisions the same kernels from its install hook.
-
 Download this model repository as an OrbitQuant artifact, then load the source Diffusers pipeline with the quantized component patched in:
 
 ```python
@@ -158,6 +147,7 @@ The matrix uses all ten prompts from `image_visual_v2` at `1024x1024`. BF16 and
 
 - Method: `orbitquant`
 - Bits: `W2A3`
+- Module weight-width counts: `W2: 44, W3: 110, W4: 84`
 - Runtime mode: `auto_fused`
 - Activation kernel backend: `auto`
 - Activation normalization epsilon: `1e-10`
@@ -179,10 +169,6 @@ The matrix uses all ten prompts from `image_visual_v2` at `1024x1024`. BF16 and
 - Calibration data: none
 - Text encoders and VAE: left in source precision by default
 
-## Observed Quality
-
-**Warning:** The OrbitQuant W2A3 outputs show severe visual collapse across all ten native comparison prompts. Use W3A3 or W4A4 for this model.
-
 ## Visual Comparison
 
 The following assets are stored in this artifact and compare the BF16 base generation against the OrbitQuant generation with the same prompt and seed.

SHA256SUMS

@@ -1,10 +1,10 @@
-81c8962c3f346c0f4032f8d718eb4e45bdebc306a1314091390ca16211e4a017  README.md
-a55ff6163946a26d6ed5f77323b5874429f822895193d3e94f2e26c57fa47081  assets/image_generation_comparison_matrix.webp
-4d46af49b4de4b6301d950a1016d67166fcf7ab6df98734031b0736bd8fd1457  benchmark/summary.json
-aa8e04810221025dc262f66632975bdbf8466b341ad19db61ba4ad4f174f402f  model.safetensors
-61be70974b031f683d3af31c38322999f93c0788909aa7e33b91499962db225a  model_index.json
-3c06a807bddc6af0246586ded2276ec8dca0e309543a9701de1b6872fb8b6b31  orbitquant_codebooks.safetensors
-ba3632cdddb971d5007dfe55bda33c131f4392e187962b616417875feb4e8294  orbitquant_manifest.json
+654a4ab7deb5ef03cc42094f0d98ddb5056e49cac58b83545b7fc2ecf214926d  README.md
+8f4235baa433900bfaf3ac5160af91c0ed68c25fffa02958f19d0a1b9dc3e421  assets/image_generation_comparison_matrix.webp
+119fadb02071c014a9c51c1108a54e20fa3532b0eaa39c2a57b788787ecbd38c  benchmark/summary.json
+667324a633b4e4d92931ca523943c093da46c24472d3f7dc315782ca57847585  model.safetensors
+cffb8406914ef65f5bb4df0c893e680c39f4c99ab4d6629f4457f61797ee33b3  model_index.json
+e7737934089114db3107ba8c16e8c6a882f259fc97d02352cf5f4fb1b0656fe9  orbitquant_codebooks.safetensors
+b11e2c4632b3d563c553eb33f8fc9162638ff19ea7c195a1b1e2cfe5dab69b17  orbitquant_manifest.json
 e1724c9394bf83ea05da6b457b8072d2bcb3d5a533dafce916da8deb517d3825  orbitquant_rotations.safetensors
-0aee42e64abd66ff9293ec140df94e62bf7836eb96a21528a728282a0278c6ed  prompts.json
-d5fbd286ff541dc629e0a2b5a216580674faaf7a3814679b0fb7e4a2122cc67b  quantization_config.json
+f48aca6d4d681d1ed10dcce8ad43fedb965b4a6a4b11003eb5e37ea53e4eb526  prompts.json
+6b778e6789a13a9e44c795571fc7d524d775730b602b634b87ba6c511d21e784  quantization_config.json

benchmark/summary.json

@@ -1,69 +1,85 @@
 {
-  "status": "native_validation_complete",
-  "source_model_id": "Tongyi-MAI/Z-Image-Turbo",
-  "source_revision": "f332072aa78be7aecdf3ee76d5c247082da564a6",
-  "weight_bits": 2,
   "activation_bits": 3,
-  "codebook_version": 2,
-  "target_policy": "z_image",
-  "runtime_mode": "auto_fused",
-  "activation_kernel_backend": "auto",
   "activation_eps": 1e-10,
+  "activation_kernel_backend": "auto",
+  "adaln_seconds": 0.14151337649673223,
+  "codebook_version": 2,
+  "device_transfer_seconds": 1.3492279867641628,
+  "module_device_transfer_count": 1,
+  "orbitquant_seconds": 1.1136700571514666,
   "quantization_device": "cuda",
-  "weight_quantization_backend": "triton_cuda",
-  "quantization_staging_mode": "streaming",
+  "quantization_elapsed_seconds": 2.619350762106478,
+  "quantization_staging_mode": "component",
+  "runtime_mode": "auto_fused",
+  "source_model_id": "Tongyi-MAI/Z-Image-Turbo",
+  "source_revision": "f332072aa78be7aecdf3ee76d5c247082da564a6",
+  "status": "metrics_recorded",
   "synchronize_per_module": false,
-  "quantization_elapsed_seconds": 3.5582019980065525,
-  "orbitquant_seconds": 2.1125443908385932,
-  "adaln_seconds": 0.015195954591035843,
-  "device_transfer_seconds": 1.4178410354070365,
-  "module_device_transfer_count": 270,
-  "metrics": {},
+  "target_policy": "z_image",
+  "weight_bits": 2,
+  "weight_quantization_backend": "triton_cuda",
+  "metrics": {
+    "original": {
+      "records": 10,
+      "latest_metrics": {
+        "generated_samples": 1,
+        "peak_vram_bytes": 23267429888,
+        "wall_time_seconds": 6.092661592178047
+      }
+    },
+    "orbitquant": {
+      "records": 10,
+      "latest_metrics": {
+        "generated_samples": 1,
+        "wall_time_seconds": 7.162837381940335,
+        "peak_vram_bytes": 13385215488
+      }
+    }
+  },
   "native_smoke": {
     "proof_format": "orbitquant-native-smoke-v1",
-    "proof_source": "native_visual_pack_v2",
     "comparison_asset_path": "assets/image_generation_comparison_matrix.webp",
     "paired_prompt_seed_count": 10,
     "paired_prompt_seed_keys": [
       [
         "z-image-native",
         "0",
-        "simple-object"
+        "color-binding"
       ],
       [
         "z-image-native",
         "0",
-        "two-object-composition"
+        "counting"
       ],
       [
         "z-image-native",
         "0",
-        "counting"
+        "cyrillic-text-rendering"
       ],
       [
         "z-image-native",
         "0",
-        "color-binding"
+        "english-text-rendering"
       ],
       [
         "z-image-native",
         "0",
-        "spatial-relationship"
+        "long-prompt"
       ],
       [
         "z-image-native",
         "0",
-        "long-prompt"
+        "occlusion-reflection"
       ],
       [
         "z-image-native",
         "0",
-        "english-text-rendering"
+        "simple-object"
       ],
       [
         "z-image-native",
         "0",
-        "cyrillic-text-rendering"
+        "spatial-relationship"
       ],
       [
         "z-image-native",
@@ -73,7 +89,7 @@
       [
         "z-image-native",
         "0",
-        "occlusion-reflection"
+        "two-object-composition"
       ]
     ],
     "splits": {
@@ -86,57 +102,57 @@
           "0"
         ],
         "prompt_ids": [
-          "simple-object",
-          "two-object-composition",
-          "counting",
           "color-binding",
-          "spatial-relationship",
-          "long-prompt",
-          "english-text-rendering",
+          "counting",
           "cyrillic-text-rendering",
+          "english-text-rendering",
+          "long-prompt",
+          "occlusion-reflection",
+          "simple-object",
+          "spatial-relationship",
           "style-heavy",
-          "occlusion-reflection"
+          "two-object-composition"
         ],
         "pair_keys": [
           [
             "z-image-native",
             "0",
-            "simple-object"
+            "color-binding"
           ],
           [
             "z-image-native",
             "0",
-            "two-object-composition"
+            "counting"
           ],
           [
             "z-image-native",
             "0",
-            "counting"
+            "cyrillic-text-rendering"
           ],
           [
             "z-image-native",
             "0",
-            "color-binding"
+            "english-text-rendering"
           ],
           [
             "z-image-native",
             "0",
-            "spatial-relationship"
+            "long-prompt"
           ],
           [
             "z-image-native",
             "0",
-            "long-prompt"
+            "occlusion-reflection"
           ],
           [
             "z-image-native",
             "0",
-            "english-text-rendering"
+            "simple-object"
           ],
           [
             "z-image-native",
             "0",
-            "cyrillic-text-rendering"
+            "spatial-relationship"
           ],
           [
             "z-image-native",
@@ -146,7 +162,7 @@
           [
             "z-image-native",
             "0",
-            "occlusion-reflection"
+            "two-object-composition"
           ]
         ],
         "native_settings": [
@@ -169,57 +185,57 @@
           "0"
         ],
         "prompt_ids": [
-          "simple-object",
-          "two-object-composition",
-          "counting",
           "color-binding",
-          "spatial-relationship",
-          "long-prompt",
-          "english-text-rendering",
+          "counting",
           "cyrillic-text-rendering",
+          "english-text-rendering",
+          "long-prompt",
+          "occlusion-reflection",
+          "simple-object",
+          "spatial-relationship",
           "style-heavy",
-          "occlusion-reflection"
+          "two-object-composition"
         ],
         "pair_keys": [
           [
             "z-image-native",
             "0",
-            "simple-object"
+            "color-binding"
           ],
           [
             "z-image-native",
             "0",
-            "two-object-composition"
+            "counting"
           ],
           [
             "z-image-native",
             "0",
-            "counting"
+            "cyrillic-text-rendering"
           ],
           [
             "z-image-native",
             "0",
-            "color-binding"
+            "english-text-rendering"
           ],
           [
             "z-image-native",
             "0",
-            "spatial-relationship"
+            "long-prompt"
           ],
           [
             "z-image-native",
             "0",
-            "long-prompt"
+            "occlusion-reflection"
           ],
           [
             "z-image-native",
             "0",
-            "english-text-rendering"
+            "simple-object"
           ],
           [
             "z-image-native",
             "0",
-            "cyrillic-text-rendering"
+            "spatial-relationship"
           ],
           [
             "z-image-native",
@@ -229,7 +245,7 @@
           [
             "z-image-native",
             "0",
-            "occlusion-reflection"
+            "two-object-composition"
           ]
         ],
         "native_settings": [
@@ -246,7 +262,5 @@
     }
   },
   "published_summary": "compact",
-  "raw_generation_records": "local-only",
-  "comparison_prompt_pack": "image_visual_v2",
-  "observed_quality": "The OrbitQuant W2A3 outputs show severe visual collapse across all ten native comparison prompts. Use W3A3 or W4A4 for this model."
+  "raw_generation_records": "local-only"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa8e04810221025dc262f66632975bdbf8466b341ad19db61ba4ad4f174f402f
-size 1599432256
+oid sha256:667324a633b4e4d92931ca523943c093da46c24472d3f7dc315782ca57847585
+size 2535778088

model_index.json

@@ -20,5 +20,5 @@
   "activation_eps": 1e-10,
   "quantization_device": "cuda",
   "weight_quantization_backend": "triton_cuda",
-  "quantization_staging_mode": "streaming"
+  "quantization_staging_mode": "component"
 }

orbitquant_codebooks.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3c06a807bddc6af0246586ded2276ec8dca0e309543a9701de1b6872fb8b6b31
-size 816
+oid sha256:e7737934089114db3107ba8c16e8c6a882f259fc97d02352cf5f4fb1b0656fe9
+size 1204

orbitquant_manifest.json

@@ -20,7 +20,7 @@
   "adaln_group_size": 64,
   "quantization_device": "cuda",
   "weight_quantization_backend": "triton_cuda",
-  "quantization_staging_mode": "streaming",
+  "quantization_staging_mode": "component",
   "target_policy": "z_image",
   "adaln_policy": "int4_rtn_group64_bf16_activation",
   "quantized_modules": [
@@ -342,43 +342,43 @@
       128
     ],
     "noise_refiner.0.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.0.attention.to_q.row_norms": [
       3840
     ],
     "noise_refiner.0.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.0.attention.to_k.row_norms": [
       3840
     ],
     "noise_refiner.0.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.0.attention.to_v.row_norms": [
       3840
     ],
     "noise_refiner.0.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.0.attention.to_out.0.row_norms": [
       3840
     ],
     "noise_refiner.0.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "noise_refiner.0.feed_forward.w1.row_norms": [
       10240
     ],
     "noise_refiner.0.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "noise_refiner.0.feed_forward.w2.row_norms": [
       3840
     ],
     "noise_refiner.0.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "noise_refiner.0.feed_forward.w3.row_norms": [
       10240
@@ -395,9 +395,6 @@
     "noise_refiner.0.ffn_norm2.weight": [
       3840
     ],
-    "noise_refiner.0.adaLN_modulation.0.bias": [
-      15360
-    ],
     "noise_refiner.0.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -405,6 +402,9 @@
       15360,
       4
     ],
+    "noise_refiner.0.adaLN_modulation.0.bias": [
+      15360
+    ],
     "noise_refiner.1.attention.norm_q.weight": [
       128
     ],
@@ -412,43 +412,43 @@
       128
     ],
     "noise_refiner.1.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.1.attention.to_q.row_norms": [
       3840
     ],
     "noise_refiner.1.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.1.attention.to_k.row_norms": [
       3840
     ],
     "noise_refiner.1.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.1.attention.to_v.row_norms": [
       3840
     ],
     "noise_refiner.1.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "noise_refiner.1.attention.to_out.0.row_norms": [
       3840
     ],
     "noise_refiner.1.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "noise_refiner.1.feed_forward.w1.row_norms": [
       10240
     ],
     "noise_refiner.1.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "noise_refiner.1.feed_forward.w2.row_norms": [
       3840
     ],
     "noise_refiner.1.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "noise_refiner.1.feed_forward.w3.row_norms": [
       10240
@@ -465,9 +465,6 @@
     "noise_refiner.1.ffn_norm2.weight": [
       3840
     ],
-    "noise_refiner.1.adaLN_modulation.0.bias": [
-      15360
-    ],
     "noise_refiner.1.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -475,6 +472,9 @@
       15360,
       4
     ],
+    "noise_refiner.1.adaLN_modulation.0.bias": [
+      15360
+    ],
     "context_refiner.0.attention.norm_q.weight": [
       128
     ],
@@ -482,43 +482,43 @@
       128
     ],
     "context_refiner.0.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.0.attention.to_q.row_norms": [
       3840
     ],
     "context_refiner.0.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.0.attention.to_k.row_norms": [
       3840
     ],
     "context_refiner.0.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.0.attention.to_v.row_norms": [
       3840
     ],
     "context_refiner.0.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.0.attention.to_out.0.row_norms": [
       3840
     ],
     "context_refiner.0.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "context_refiner.0.feed_forward.w1.row_norms": [
       10240
     ],
     "context_refiner.0.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "context_refiner.0.feed_forward.w2.row_norms": [
       3840
     ],
     "context_refiner.0.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "context_refiner.0.feed_forward.w3.row_norms": [
       10240
@@ -542,43 +542,43 @@
       128
     ],
     "context_refiner.1.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.1.attention.to_q.row_norms": [
       3840
     ],
     "context_refiner.1.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.1.attention.to_k.row_norms": [
       3840
     ],
     "context_refiner.1.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.1.attention.to_v.row_norms": [
       3840
     ],
     "context_refiner.1.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "context_refiner.1.attention.to_out.0.row_norms": [
       3840
     ],
     "context_refiner.1.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "context_refiner.1.feed_forward.w1.row_norms": [
       10240
     ],
     "context_refiner.1.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "context_refiner.1.feed_forward.w2.row_norms": [
       3840
     ],
     "context_refiner.1.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "context_refiner.1.feed_forward.w3.row_norms": [
       10240
@@ -626,43 +626,43 @@
       128
     ],
     "layers.0.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.0.attention.to_q.row_norms": [
       3840
     ],
     "layers.0.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.0.attention.to_k.row_norms": [
       3840
     ],
     "layers.0.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.0.attention.to_v.row_norms": [
       3840
     ],
     "layers.0.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.0.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.0.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.0.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.0.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.0.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.0.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.0.feed_forward.w3.row_norms": [
       10240
@@ -679,9 +679,6 @@
     "layers.0.ffn_norm2.weight": [
       3840
     ],
-    "layers.0.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.0.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -689,6 +686,9 @@
       15360,
       4
     ],
+    "layers.0.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.1.attention.norm_q.weight": [
       128
     ],
@@ -696,43 +696,43 @@
       128
     ],
     "layers.1.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.1.attention.to_q.row_norms": [
       3840
     ],
     "layers.1.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.1.attention.to_k.row_norms": [
       3840
     ],
     "layers.1.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.1.attention.to_v.row_norms": [
       3840
     ],
     "layers.1.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.1.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.1.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.1.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.1.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.1.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.1.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.1.feed_forward.w3.row_norms": [
       10240
@@ -749,9 +749,6 @@
     "layers.1.ffn_norm2.weight": [
       3840
     ],
-    "layers.1.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.1.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -759,6 +756,9 @@
       15360,
       4
     ],
+    "layers.1.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.2.attention.norm_q.weight": [
       128
     ],
@@ -766,43 +766,43 @@
       128
     ],
     "layers.2.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.2.attention.to_q.row_norms": [
       3840
     ],
     "layers.2.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.2.attention.to_k.row_norms": [
       3840
     ],
     "layers.2.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.2.attention.to_v.row_norms": [
       3840
     ],
     "layers.2.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.2.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.2.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.2.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.2.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.2.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.2.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.2.feed_forward.w3.row_norms": [
       10240
@@ -819,9 +819,6 @@
     "layers.2.ffn_norm2.weight": [
       3840
     ],
-    "layers.2.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.2.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -829,6 +826,9 @@
       15360,
       4
     ],
+    "layers.2.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.3.attention.norm_q.weight": [
       128
     ],
@@ -836,43 +836,43 @@
       128
     ],
     "layers.3.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.3.attention.to_q.row_norms": [
       3840
     ],
     "layers.3.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.3.attention.to_k.row_norms": [
       3840
     ],
     "layers.3.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.3.attention.to_v.row_norms": [
       3840
     ],
     "layers.3.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.3.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.3.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.3.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.3.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.3.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.3.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.3.feed_forward.w3.row_norms": [
       10240
@@ -889,9 +889,6 @@
     "layers.3.ffn_norm2.weight": [
       3840
     ],
-    "layers.3.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.3.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -899,6 +896,9 @@
       15360,
       4
     ],
+    "layers.3.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.4.attention.norm_q.weight": [
       128
     ],
@@ -918,31 +918,31 @@
       3840
     ],
     "layers.4.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.4.attention.to_v.row_norms": [
       3840
     ],
     "layers.4.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.4.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.4.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.4.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.4.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.4.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.4.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.4.feed_forward.w3.row_norms": [
       10240
@@ -959,9 +959,6 @@
     "layers.4.ffn_norm2.weight": [
       3840
     ],
-    "layers.4.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.4.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -969,6 +966,9 @@
       15360,
       4
     ],
+    "layers.4.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.5.attention.norm_q.weight": [
       128
     ],
@@ -988,31 +988,31 @@
       3840
     ],
     "layers.5.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.5.attention.to_v.row_norms": [
       3840
     ],
     "layers.5.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.5.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.5.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.5.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.5.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.5.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.5.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.5.feed_forward.w3.row_norms": [
       10240
@@ -1029,9 +1029,6 @@
     "layers.5.ffn_norm2.weight": [
       3840
     ],
-    "layers.5.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.5.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1039,6 +1036,9 @@
       15360,
       4
     ],
+    "layers.5.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.6.attention.norm_q.weight": [
       128
     ],
@@ -1058,31 +1058,31 @@
       3840
     ],
     "layers.6.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.6.attention.to_v.row_norms": [
       3840
     ],
     "layers.6.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.6.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.6.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.6.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.6.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.6.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.6.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.6.feed_forward.w3.row_norms": [
       10240
@@ -1099,9 +1099,6 @@
     "layers.6.ffn_norm2.weight": [
       3840
     ],
-    "layers.6.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.6.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1109,6 +1106,9 @@
       15360,
       4
     ],
+    "layers.6.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.7.attention.norm_q.weight": [
       128
     ],
@@ -1128,31 +1128,31 @@
       3840
     ],
     "layers.7.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.7.attention.to_v.row_norms": [
       3840
     ],
     "layers.7.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.7.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.7.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.7.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.7.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.7.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.7.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.7.feed_forward.w3.row_norms": [
       10240
@@ -1169,9 +1169,6 @@
     "layers.7.ffn_norm2.weight": [
       3840
     ],
-    "layers.7.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.7.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1179,6 +1176,9 @@
       15360,
       4
     ],
+    "layers.7.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.8.attention.norm_q.weight": [
       128
     ],
@@ -1198,31 +1198,31 @@
       3840
     ],
     "layers.8.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.8.attention.to_v.row_norms": [
       3840
     ],
     "layers.8.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.8.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.8.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.8.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.8.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.8.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.8.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.8.feed_forward.w3.row_norms": [
       10240
@@ -1239,9 +1239,6 @@
     "layers.8.ffn_norm2.weight": [
       3840
     ],
-    "layers.8.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.8.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1249,6 +1246,9 @@
       15360,
       4
     ],
+    "layers.8.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.9.attention.norm_q.weight": [
       128
     ],
@@ -1268,31 +1268,31 @@
       3840
     ],
     "layers.9.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.9.attention.to_v.row_norms": [
       3840
     ],
     "layers.9.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.9.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.9.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.9.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.9.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.9.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.9.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.9.feed_forward.w3.row_norms": [
       10240
@@ -1309,9 +1309,6 @@
     "layers.9.ffn_norm2.weight": [
       3840
     ],
-    "layers.9.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.9.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1319,6 +1316,9 @@
       15360,
       4
     ],
+    "layers.9.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.10.attention.norm_q.weight": [
       128
     ],
@@ -1338,31 +1338,31 @@
       3840
     ],
     "layers.10.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.10.attention.to_v.row_norms": [
       3840
     ],
     "layers.10.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.10.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.10.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.10.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.10.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.10.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.10.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.10.feed_forward.w3.row_norms": [
       10240
@@ -1379,9 +1379,6 @@
     "layers.10.ffn_norm2.weight": [
       3840
     ],
-    "layers.10.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.10.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1389,6 +1386,9 @@
       15360,
       4
     ],
+    "layers.10.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.11.attention.norm_q.weight": [
       128
     ],
@@ -1408,31 +1408,31 @@
       3840
     ],
     "layers.11.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.11.attention.to_v.row_norms": [
       3840
     ],
     "layers.11.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.11.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.11.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.11.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.11.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.11.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.11.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.11.feed_forward.w3.row_norms": [
       10240
@@ -1449,9 +1449,6 @@
     "layers.11.ffn_norm2.weight": [
       3840
     ],
-    "layers.11.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.11.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1459,6 +1456,9 @@
       15360,
       4
     ],
+    "layers.11.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.12.attention.norm_q.weight": [
       128
     ],
@@ -1478,31 +1478,31 @@
       3840
     ],
     "layers.12.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.12.attention.to_v.row_norms": [
       3840
     ],
     "layers.12.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.12.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.12.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.12.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.12.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.12.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.12.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.12.feed_forward.w3.row_norms": [
       10240
@@ -1519,9 +1519,6 @@
     "layers.12.ffn_norm2.weight": [
       3840
     ],
-    "layers.12.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.12.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1529,6 +1526,9 @@
       15360,
       4
     ],
+    "layers.12.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.13.attention.norm_q.weight": [
       128
     ],
@@ -1548,31 +1548,31 @@
       3840
     ],
     "layers.13.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.13.attention.to_v.row_norms": [
       3840
     ],
     "layers.13.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.13.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.13.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.13.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.13.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.13.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.13.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.13.feed_forward.w3.row_norms": [
       10240
@@ -1589,9 +1589,6 @@
     "layers.13.ffn_norm2.weight": [
       3840
     ],
-    "layers.13.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.13.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1599,6 +1596,9 @@
       15360,
       4
     ],
+    "layers.13.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.14.attention.norm_q.weight": [
       128
     ],
@@ -1618,31 +1618,31 @@
       3840
     ],
     "layers.14.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.14.attention.to_v.row_norms": [
       3840
     ],
     "layers.14.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.14.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.14.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.14.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.14.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.14.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.14.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.14.feed_forward.w3.row_norms": [
       10240
@@ -1659,9 +1659,6 @@
     "layers.14.ffn_norm2.weight": [
       3840
     ],
-    "layers.14.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.14.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1669,6 +1666,9 @@
       15360,
       4
     ],
+    "layers.14.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.15.attention.norm_q.weight": [
       128
     ],
@@ -1688,31 +1688,31 @@
       3840
     ],
     "layers.15.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.15.attention.to_v.row_norms": [
       3840
     ],
     "layers.15.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.15.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.15.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.15.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.15.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.15.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.15.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.15.feed_forward.w3.row_norms": [
       10240
@@ -1729,9 +1729,6 @@
     "layers.15.ffn_norm2.weight": [
       3840
     ],
-    "layers.15.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.15.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1739,6 +1736,9 @@
       15360,
       4
     ],
+    "layers.15.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.16.attention.norm_q.weight": [
       128
     ],
@@ -1758,31 +1758,31 @@
       3840
     ],
     "layers.16.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.16.attention.to_v.row_norms": [
       3840
     ],
     "layers.16.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.16.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.16.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.16.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.16.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.16.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.16.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.16.feed_forward.w3.row_norms": [
       10240
@@ -1799,9 +1799,6 @@
     "layers.16.ffn_norm2.weight": [
       3840
     ],
-    "layers.16.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.16.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1809,6 +1806,9 @@
       15360,
       4
     ],
+    "layers.16.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.17.attention.norm_q.weight": [
       128
     ],
@@ -1828,31 +1828,31 @@
       3840
     ],
     "layers.17.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.17.attention.to_v.row_norms": [
       3840
     ],
     "layers.17.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.17.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.17.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.17.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.17.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.17.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.17.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.17.feed_forward.w3.row_norms": [
       10240
@@ -1869,9 +1869,6 @@
     "layers.17.ffn_norm2.weight": [
       3840
     ],
-    "layers.17.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.17.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1879,6 +1876,9 @@
       15360,
       4
     ],
+    "layers.17.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.18.attention.norm_q.weight": [
       128
     ],
@@ -1898,31 +1898,31 @@
       3840
     ],
     "layers.18.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.18.attention.to_v.row_norms": [
       3840
     ],
     "layers.18.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.18.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.18.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.18.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.18.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.18.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.18.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.18.feed_forward.w3.row_norms": [
       10240
@@ -1939,9 +1939,6 @@
     "layers.18.ffn_norm2.weight": [
       3840
     ],
-    "layers.18.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.18.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -1949,6 +1946,9 @@
       15360,
       4
     ],
+    "layers.18.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.19.attention.norm_q.weight": [
       128
     ],
@@ -1968,31 +1968,31 @@
       3840
     ],
     "layers.19.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.19.attention.to_v.row_norms": [
       3840
     ],
     "layers.19.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.19.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.19.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.19.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.19.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.19.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.19.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.19.feed_forward.w3.row_norms": [
       10240
@@ -2009,9 +2009,6 @@
     "layers.19.ffn_norm2.weight": [
       3840
     ],
-    "layers.19.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.19.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2019,6 +2016,9 @@
       15360,
       4
     ],
+    "layers.19.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.20.attention.norm_q.weight": [
       128
     ],
@@ -2038,31 +2038,31 @@
       3840
     ],
     "layers.20.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.20.attention.to_v.row_norms": [
       3840
     ],
     "layers.20.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.20.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.20.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.20.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.20.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.20.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.20.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.20.feed_forward.w3.row_norms": [
       10240
@@ -2079,9 +2079,6 @@
     "layers.20.ffn_norm2.weight": [
       3840
     ],
-    "layers.20.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.20.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2089,6 +2086,9 @@
       15360,
       4
     ],
+    "layers.20.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.21.attention.norm_q.weight": [
       128
     ],
@@ -2108,31 +2108,31 @@
       3840
     ],
     "layers.21.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.21.attention.to_v.row_norms": [
       3840
     ],
     "layers.21.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.21.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.21.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.21.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.21.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.21.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.21.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.21.feed_forward.w3.row_norms": [
       10240
@@ -2149,9 +2149,6 @@
     "layers.21.ffn_norm2.weight": [
       3840
     ],
-    "layers.21.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.21.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2159,6 +2156,9 @@
       15360,
       4
     ],
+    "layers.21.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.22.attention.norm_q.weight": [
       128
     ],
@@ -2178,31 +2178,31 @@
       3840
     ],
     "layers.22.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.22.attention.to_v.row_norms": [
       3840
     ],
     "layers.22.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.22.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.22.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.22.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.22.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.22.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.22.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.22.feed_forward.w3.row_norms": [
       10240
@@ -2219,9 +2219,6 @@
     "layers.22.ffn_norm2.weight": [
       3840
     ],
-    "layers.22.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.22.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2229,6 +2226,9 @@
       15360,
       4
     ],
+    "layers.22.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.23.attention.norm_q.weight": [
       128
     ],
@@ -2248,31 +2248,31 @@
       3840
     ],
     "layers.23.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.23.attention.to_v.row_norms": [
       3840
     ],
     "layers.23.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.23.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.23.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.23.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.23.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.23.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.23.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.23.feed_forward.w3.row_norms": [
       10240
@@ -2289,9 +2289,6 @@
     "layers.23.ffn_norm2.weight": [
       3840
     ],
-    "layers.23.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.23.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2299,6 +2296,9 @@
       15360,
       4
     ],
+    "layers.23.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.24.attention.norm_q.weight": [
       128
     ],
@@ -2318,31 +2318,31 @@
       3840
     ],
     "layers.24.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.24.attention.to_v.row_norms": [
       3840
     ],
     "layers.24.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.24.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.24.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.24.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.24.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.24.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.24.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.24.feed_forward.w3.row_norms": [
       10240
@@ -2359,9 +2359,6 @@
     "layers.24.ffn_norm2.weight": [
       3840
     ],
-    "layers.24.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.24.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2369,6 +2366,9 @@
       15360,
       4
     ],
+    "layers.24.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.25.attention.norm_q.weight": [
       128
     ],
@@ -2388,31 +2388,31 @@
       3840
     ],
     "layers.25.attention.to_v.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.25.attention.to_v.row_norms": [
       3840
     ],
     "layers.25.attention.to_out.0.packed_weight_indices": [
-      3686400
+      5529600
     ],
     "layers.25.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.25.feed_forward.w1.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.25.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.25.feed_forward.w2.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.25.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.25.feed_forward.w3.packed_weight_indices": [
-      9830400
+      14745600
     ],
     "layers.25.feed_forward.w3.row_norms": [
       10240
@@ -2429,9 +2429,6 @@
     "layers.25.ffn_norm2.weight": [
       3840
     ],
-    "layers.25.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.25.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2439,6 +2436,9 @@
       15360,
       4
     ],
+    "layers.25.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.26.attention.norm_q.weight": [
       128
     ],
@@ -2446,43 +2446,43 @@
       128
     ],
     "layers.26.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.26.attention.to_q.row_norms": [
       3840
     ],
     "layers.26.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.26.attention.to_k.row_norms": [
       3840
     ],
     "layers.26.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.26.attention.to_v.row_norms": [
       3840
     ],
     "layers.26.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.26.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.26.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.26.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.26.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.26.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.26.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.26.feed_forward.w3.row_norms": [
       10240
@@ -2499,9 +2499,6 @@
     "layers.26.ffn_norm2.weight": [
       3840
     ],
-    "layers.26.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.26.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2509,6 +2506,9 @@
       15360,
       4
     ],
+    "layers.26.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.27.attention.norm_q.weight": [
       128
     ],
@@ -2516,43 +2516,43 @@
       128
     ],
     "layers.27.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.27.attention.to_q.row_norms": [
       3840
     ],
     "layers.27.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.27.attention.to_k.row_norms": [
       3840
     ],
     "layers.27.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.27.attention.to_v.row_norms": [
       3840
     ],
     "layers.27.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.27.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.27.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.27.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.27.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.27.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.27.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.27.feed_forward.w3.row_norms": [
       10240
@@ -2569,9 +2569,6 @@
     "layers.27.ffn_norm2.weight": [
       3840
     ],
-    "layers.27.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.27.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2579,6 +2576,9 @@
       15360,
       4
     ],
+    "layers.27.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.28.attention.norm_q.weight": [
       128
     ],
@@ -2586,43 +2586,43 @@
       128
     ],
     "layers.28.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.28.attention.to_q.row_norms": [
       3840
     ],
     "layers.28.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.28.attention.to_k.row_norms": [
       3840
     ],
     "layers.28.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.28.attention.to_v.row_norms": [
       3840
     ],
     "layers.28.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.28.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.28.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.28.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.28.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.28.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.28.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.28.feed_forward.w3.row_norms": [
       10240
@@ -2639,9 +2639,6 @@
     "layers.28.ffn_norm2.weight": [
       3840
     ],
-    "layers.28.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.28.adaLN_modulation.0.packed_weight": [
       1966080
     ],
@@ -2649,6 +2646,9 @@
       15360,
       4
     ],
+    "layers.28.adaLN_modulation.0.bias": [
+      15360
+    ],
     "layers.29.attention.norm_q.weight": [
       128
     ],
@@ -2656,43 +2656,43 @@
       128
     ],
     "layers.29.attention.to_q.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.29.attention.to_q.row_norms": [
       3840
     ],
     "layers.29.attention.to_k.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.29.attention.to_k.row_norms": [
       3840
     ],
     "layers.29.attention.to_v.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.29.attention.to_v.row_norms": [
       3840
     ],
     "layers.29.attention.to_out.0.packed_weight_indices": [
-      3686400
+      7372800
     ],
     "layers.29.attention.to_out.0.row_norms": [
       3840
     ],
     "layers.29.feed_forward.w1.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.29.feed_forward.w1.row_norms": [
       10240
     ],
     "layers.29.feed_forward.w2.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.29.feed_forward.w2.row_norms": [
       3840
     ],
     "layers.29.feed_forward.w3.packed_weight_indices": [
-      9830400
+      19660800
     ],
     "layers.29.feed_forward.w3.row_norms": [
       10240
@@ -2709,25 +2709,221 @@
     "layers.29.ffn_norm2.weight": [
       3840
     ],
-    "layers.29.adaLN_modulation.0.bias": [
-      15360
-    ],
     "layers.29.adaLN_modulation.0.packed_weight": [
       1966080
     ],
     "layers.29.adaLN_modulation.0.scales": [
       15360,
       4
+    ],
+    "layers.29.adaLN_modulation.0.bias": [
+      15360
     ]
   },
+  "module_bits": {
+    "noise_refiner.0.attention.to_q": 4,
+    "noise_refiner.0.attention.to_k": 4,
+    "noise_refiner.0.attention.to_v": 4,
+    "noise_refiner.0.attention.to_out.0": 4,
+    "noise_refiner.0.feed_forward.w1": 4,
+    "noise_refiner.0.feed_forward.w2": 4,
+    "noise_refiner.0.feed_forward.w3": 4,
+    "noise_refiner.1.attention.to_q": 4,
+    "noise_refiner.1.attention.to_k": 4,
+    "noise_refiner.1.attention.to_v": 4,
+    "noise_refiner.1.attention.to_out.0": 4,
+    "noise_refiner.1.feed_forward.w1": 4,
+    "noise_refiner.1.feed_forward.w2": 4,
+    "noise_refiner.1.feed_forward.w3": 4,
+    "context_refiner.0.attention.to_q": 4,
+    "context_refiner.0.attention.to_k": 4,
+    "context_refiner.0.attention.to_v": 4,
+    "context_refiner.0.attention.to_out.0": 4,
+    "context_refiner.0.feed_forward.w1": 4,
+    "context_refiner.0.feed_forward.w2": 4,
+    "context_refiner.0.feed_forward.w3": 4,
+    "context_refiner.1.attention.to_q": 4,
+    "context_refiner.1.attention.to_k": 4,
+    "context_refiner.1.attention.to_v": 4,
+    "context_refiner.1.attention.to_out.0": 4,
+    "context_refiner.1.feed_forward.w1": 4,
+    "context_refiner.1.feed_forward.w2": 4,
+    "context_refiner.1.feed_forward.w3": 4,
+    "layers.0.attention.to_q": 4,
+    "layers.0.attention.to_k": 4,
+    "layers.0.attention.to_v": 4,
+    "layers.0.attention.to_out.0": 4,
+    "layers.0.feed_forward.w1": 4,
+    "layers.0.feed_forward.w2": 4,
+    "layers.0.feed_forward.w3": 4,
+    "layers.1.attention.to_q": 4,
+    "layers.1.attention.to_k": 4,
+    "layers.1.attention.to_v": 4,
+    "layers.1.attention.to_out.0": 4,
+    "layers.1.feed_forward.w1": 4,
+    "layers.1.feed_forward.w2": 4,
+    "layers.1.feed_forward.w3": 4,
+    "layers.2.attention.to_q": 4,
+    "layers.2.attention.to_k": 4,
+    "layers.2.attention.to_v": 4,
+    "layers.2.attention.to_out.0": 4,
+    "layers.2.feed_forward.w1": 4,
+    "layers.2.feed_forward.w2": 4,
+    "layers.2.feed_forward.w3": 4,
+    "layers.3.attention.to_q": 4,
+    "layers.3.attention.to_k": 4,
+    "layers.3.attention.to_v": 4,
+    "layers.3.attention.to_out.0": 4,
+    "layers.3.feed_forward.w1": 4,
+    "layers.3.feed_forward.w2": 4,
+    "layers.3.feed_forward.w3": 4,
+    "layers.4.attention.to_v": 3,
+    "layers.4.attention.to_out.0": 3,
+    "layers.4.feed_forward.w1": 3,
+    "layers.4.feed_forward.w2": 3,
+    "layers.4.feed_forward.w3": 3,
+    "layers.5.attention.to_v": 3,
+    "layers.5.attention.to_out.0": 3,
+    "layers.5.feed_forward.w1": 3,
+    "layers.5.feed_forward.w2": 3,
+    "layers.5.feed_forward.w3": 3,
+    "layers.6.attention.to_v": 3,
+    "layers.6.attention.to_out.0": 3,
+    "layers.6.feed_forward.w1": 3,
+    "layers.6.feed_forward.w2": 3,
+    "layers.6.feed_forward.w3": 3,
+    "layers.7.attention.to_v": 3,
+    "layers.7.attention.to_out.0": 3,
+    "layers.7.feed_forward.w1": 3,
+    "layers.7.feed_forward.w2": 3,
+    "layers.7.feed_forward.w3": 3,
+    "layers.8.attention.to_v": 3,
+    "layers.8.attention.to_out.0": 3,
+    "layers.8.feed_forward.w1": 3,
+    "layers.8.feed_forward.w2": 3,
+    "layers.8.feed_forward.w3": 3,
+    "layers.9.attention.to_v": 3,
+    "layers.9.attention.to_out.0": 3,
+    "layers.9.feed_forward.w1": 3,
+    "layers.9.feed_forward.w2": 3,
+    "layers.9.feed_forward.w3": 3,
+    "layers.10.attention.to_v": 3,
+    "layers.10.attention.to_out.0": 3,
+    "layers.10.feed_forward.w1": 3,
+    "layers.10.feed_forward.w2": 3,
+    "layers.10.feed_forward.w3": 3,
+    "layers.11.attention.to_v": 3,
+    "layers.11.attention.to_out.0": 3,
+    "layers.11.feed_forward.w1": 3,
+    "layers.11.feed_forward.w2": 3,
+    "layers.11.feed_forward.w3": 3,
+    "layers.12.attention.to_v": 3,
+    "layers.12.attention.to_out.0": 3,
+    "layers.12.feed_forward.w1": 3,
+    "layers.12.feed_forward.w2": 3,
+    "layers.12.feed_forward.w3": 3,
+    "layers.13.attention.to_v": 3,
+    "layers.13.attention.to_out.0": 3,
+    "layers.13.feed_forward.w1": 3,
+    "layers.13.feed_forward.w2": 3,
+    "layers.13.feed_forward.w3": 3,
+    "layers.14.attention.to_v": 3,
+    "layers.14.attention.to_out.0": 3,
+    "layers.14.feed_forward.w1": 3,
+    "layers.14.feed_forward.w2": 3,
+    "layers.14.feed_forward.w3": 3,
+    "layers.15.attention.to_v": 3,
+    "layers.15.attention.to_out.0": 3,
+    "layers.15.feed_forward.w1": 3,
+    "layers.15.feed_forward.w2": 3,
+    "layers.15.feed_forward.w3": 3,
+    "layers.16.attention.to_v": 3,
+    "layers.16.attention.to_out.0": 3,
+    "layers.16.feed_forward.w1": 3,
+    "layers.16.feed_forward.w2": 3,
+    "layers.16.feed_forward.w3": 3,
+    "layers.17.attention.to_v": 3,
+    "layers.17.attention.to_out.0": 3,
+    "layers.17.feed_forward.w1": 3,
+    "layers.17.feed_forward.w2": 3,
+    "layers.17.feed_forward.w3": 3,
+    "layers.18.attention.to_v": 3,
+    "layers.18.attention.to_out.0": 3,
+    "layers.18.feed_forward.w1": 3,
+    "layers.18.feed_forward.w2": 3,
+    "layers.18.feed_forward.w3": 3,
+    "layers.19.attention.to_v": 3,
+    "layers.19.attention.to_out.0": 3,
+    "layers.19.feed_forward.w1": 3,
+    "layers.19.feed_forward.w2": 3,
+    "layers.19.feed_forward.w3": 3,
+    "layers.20.attention.to_v": 3,
+    "layers.20.attention.to_out.0": 3,
+    "layers.20.feed_forward.w1": 3,
+    "layers.20.feed_forward.w2": 3,
+    "layers.20.feed_forward.w3": 3,
+    "layers.21.attention.to_v": 3,
+    "layers.21.attention.to_out.0": 3,
+    "layers.21.feed_forward.w1": 3,
+    "layers.21.feed_forward.w2": 3,
+    "layers.21.feed_forward.w3": 3,
+    "layers.22.attention.to_v": 3,
+    "layers.22.attention.to_out.0": 3,
+    "layers.22.feed_forward.w1": 3,
+    "layers.22.feed_forward.w2": 3,
+    "layers.22.feed_forward.w3": 3,
+    "layers.23.attention.to_v": 3,
+    "layers.23.attention.to_out.0": 3,
+    "layers.23.feed_forward.w1": 3,
+    "layers.23.feed_forward.w2": 3,
+    "layers.23.feed_forward.w3": 3,
+    "layers.24.attention.to_v": 3,
+    "layers.24.attention.to_out.0": 3,
+    "layers.24.feed_forward.w1": 3,
+    "layers.24.feed_forward.w2": 3,
+    "layers.24.feed_forward.w3": 3,
+    "layers.25.attention.to_v": 3,
+    "layers.25.attention.to_out.0": 3,
+    "layers.25.feed_forward.w1": 3,
+    "layers.25.feed_forward.w2": 3,
+    "layers.25.feed_forward.w3": 3,
+    "layers.26.attention.to_q": 4,
+    "layers.26.attention.to_k": 4,
+    "layers.26.attention.to_v": 4,
+    "layers.26.attention.to_out.0": 4,
+    "layers.26.feed_forward.w1": 4,
+    "layers.26.feed_forward.w2": 4,
+    "layers.26.feed_forward.w3": 4,
+    "layers.27.attention.to_q": 4,
+    "layers.27.attention.to_k": 4,
+    "layers.27.attention.to_v": 4,
+    "layers.27.attention.to_out.0": 4,
+    "layers.27.feed_forward.w1": 4,
+    "layers.27.feed_forward.w2": 4,
+    "layers.27.feed_forward.w3": 4,
+    "layers.28.attention.to_q": 4,
+    "layers.28.attention.to_k": 4,
+    "layers.28.attention.to_v": 4,
+    "layers.28.attention.to_out.0": 4,
+    "layers.28.feed_forward.w1": 4,
+    "layers.28.feed_forward.w2": 4,
+    "layers.28.feed_forward.w3": 4,
+    "layers.29.attention.to_q": 4,
+    "layers.29.attention.to_k": 4,
+    "layers.29.attention.to_v": 4,
+    "layers.29.attention.to_out.0": 4,
+    "layers.29.feed_forward.w1": 4,
+    "layers.29.feed_forward.w2": 4,
+    "layers.29.feed_forward.w3": 4
+  },
   "checksums": {
-    "assets/image_generation_comparison_matrix.webp": "a55ff6163946a26d6ed5f77323b5874429f822895193d3e94f2e26c57fa47081",
-    "benchmark/summary.json": "4d46af49b4de4b6301d950a1016d67166fcf7ab6df98734031b0736bd8fd1457",
-    "model.safetensors": "aa8e04810221025dc262f66632975bdbf8466b341ad19db61ba4ad4f174f402f",
-    "model_index.json": "61be70974b031f683d3af31c38322999f93c0788909aa7e33b91499962db225a",
-    "orbitquant_codebooks.safetensors": "3c06a807bddc6af0246586ded2276ec8dca0e309543a9701de1b6872fb8b6b31",
+    "assets/image_generation_comparison_matrix.webp": "8f4235baa433900bfaf3ac5160af91c0ed68c25fffa02958f19d0a1b9dc3e421",
+    "benchmark/summary.json": "119fadb02071c014a9c51c1108a54e20fa3532b0eaa39c2a57b788787ecbd38c",
+    "model.safetensors": "667324a633b4e4d92931ca523943c093da46c24472d3f7dc315782ca57847585",
+    "model_index.json": "cffb8406914ef65f5bb4df0c893e680c39f4c99ab4d6629f4457f61797ee33b3",
+    "orbitquant_codebooks.safetensors": "e7737934089114db3107ba8c16e8c6a882f259fc97d02352cf5f4fb1b0656fe9",
     "orbitquant_rotations.safetensors": "e1724c9394bf83ea05da6b457b8072d2bcb3d5a533dafce916da8deb517d3825",
-    "prompts.json": "0aee42e64abd66ff9293ec140df94e62bf7836eb96a21528a728282a0278c6ed",
-    "quantization_config.json": "d5fbd286ff541dc629e0a2b5a216580674faaf7a3814679b0fb7e4a2122cc67b"
+    "prompts.json": "f48aca6d4d681d1ed10dcce8ad43fedb965b4a6a4b11003eb5e37ea53e4eb526",
+    "quantization_config.json": "6b778e6789a13a9e44c795571fc7d524d775730b602b634b87ba6c511d21e784"
   }
 }

prompts.json

@@ -49,19 +49,19 @@
       "id": "cyrillic-text-rendering",
       "title": "08 Russian Constructivist typography",
       "category": "russian_typography",
-      "prompt": "A richly detailed Russian Constructivist science-fiction poster with the exact Cyrillic headline \"КВАНТОВАЯ ОРБИТА\", the exact subtitle \"МОСКВА 2049\", and a small exact stamp \"КВАНТОВАНИЕ\"; diagonal red and black geometry, cream paper, cosmonaut portrait, orbital diagrams, halftone grain, folded corners, layered ink, museum archival photograph, all letters crisp and correctly ordered"
+      "prompt": "A richly detailed Russian Constructivist science-fiction poster with the exact Cyrillic headline \"\u041a\u0412\u0410\u041d\u0422\u041e\u0412\u0410\u042f \u041e\u0420\u0411\u0418\u0422\u0410\", the exact subtitle \"\u041c\u041e\u0421\u041a\u0412\u0410 2049\", and a small exact stamp \"\u041a\u0412\u0410\u041d\u0422\u041e\u0412\u0410\u041d\u0418\u0415\"; diagonal red and black geometry, cream paper, cosmonaut portrait, orbital diagrams, halftone grain, folded corners, layered ink, museum archival photograph, all letters crisp and correctly ordered"
     },
     {
       "id": "style-heavy",
       "title": "09 Japanese typography and mixed style",
       "category": "japanese_typography",
-      "prompt": "An elaborate Japanese art magazine cover combining Edo woodblock printing with a futuristic Tokyo skyline, with the exact vertical title \"量子の軌道\" and the exact subtitle \"東京の未来\"; giant indigo waves curl around glass towers, red-crowned cranes cross a gold moon, tiny pedestrians and trains fill the lower streets, visible washi fibers, layered spot colors, precise Japanese glyphs, balanced editorial layout"
+      "prompt": "An elaborate Japanese art magazine cover combining Edo woodblock printing with a futuristic Tokyo skyline, with the exact vertical title \"\u91cf\u5b50\u306e\u8ecc\u9053\" and the exact subtitle \"\u6771\u4eac\u306e\u672a\u6765\"; giant indigo waves curl around glass towers, red-crowned cranes cross a gold moon, tiny pedestrians and trains fill the lower streets, visible washi fibers, layered spot colors, precise Japanese glyphs, balanced editorial layout"
     },
     {
       "id": "occlusion-reflection",
       "title": "10 Chinese typography, reflection, occlusion",
       "category": "chinese_typography_reflection_occlusion",
-      "prompt": "A luxurious Chinese retro-futurist department-store window at night with the exact gold title \"量子轨道\" and the exact red subtitle \"未来之城\"; a curved chrome robot is partly occluded by peonies and blue-and-white porcelain, the calligraphy and neon street must appear coherently reflected across its body, multiple glass layers, silk textures, passing bicycles, cinematic rain, fine product-photography detail"
+      "prompt": "A luxurious Chinese retro-futurist department-store window at night with the exact gold title \"\u91cf\u5b50\u8f68\u9053\" and the exact red subtitle \"\u672a\u6765\u4e4b\u57ce\"; a curved chrome robot is partly occluded by peonies and blue-and-white porcelain, the calligraphy and neon street must appear coherently reflected across its body, multiple glass layers, silk textures, passing bicycles, cinematic rain, fine product-photography detail"
     }
   ]
 }

quantization_config.json

@@ -15,6 +15,8 @@
   "target_policy": "z_image",
   "adaln_policy": "int4_rtn",
   "adaln_group_size": 64,
+  "modules_to_convert": [],
+  "modules_to_use_adaln": [],
   "modules_to_not_convert": [],
   "modules_dtype_dict": {},
   "artifact_format_version": 1,
@@ -23,5 +25,11 @@
   "packed_matmul_block_m": 64,
   "packed_matmul_block_n": 64,
   "packed_matmul_block_k": 128,
-  "packed_matmul_num_warps": 4
+  "packed_matmul_num_warps": 4,
+  "weight_row_tile_size": 256,
+  "w4a4_int8_weight_cache": false,
+  "lowbit_boundary_protection": "auto",
+  "lowbit_protected_blocks": 4,
+  "lowbit_protected_bits": 4,
+  "lowbit_interior_protection": true
 }