Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.meta/create_model.py

@@ -0,0 +1,140 @@
+def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
+    import copy
+    import json
+    from pathlib import Path
+
+    import torch
+    from huggingface_hub import hf_hub_download
+    from safetensors.torch import save_file
+    from transformers import AutoConfig, AutoModelForImageTextToText, set_seed
+
+    folder = Path(save_folder)
+    folder.mkdir(parents=True, exist_ok=True)
+    # Transformers 5.18 loads this custom Loop/mHC wrapper around the native Qwen3.5 backbone; ship it pinned.
+    for name in (
+        "modeling_sensenova_loop.py",
+        "chat_template.jinja",
+        "processor_config.json",
+        "preprocessor_config.json",
+        "video_preprocessor_config.json",
+        "tokenizer.json",
+        "tokenizer_config.json",
+    ):
+        (folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
+
+    config = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
+    # Pre-loop 0-3, looped block 4-7 executed twice, post-loop 8-11: twelve physical layers / sixteen executions,
+    # four complete linear/linear/linear/full cycles. Residual width 8, head dim 32 with eight Q / four KV heads.
+    text_config = config["text_config"]
+    text_config.update(
+        head_dim=32,
+        hidden_size=8,
+        intermediate_size=32,
+        num_hidden_layers=12,
+        num_attention_heads=8,
+        num_key_value_heads=4,
+        linear_key_head_dim=32,
+        linear_num_key_heads=4,
+        linear_num_value_heads=8,
+        linear_value_head_dim=32,
+    )
+    text_config["layer_types"] = ["linear_attention", "linear_attention", "linear_attention", "full_attention"] * 3
+    text_config["rope_parameters"]["mrope_section"] = [1, 1, 2]
+    # Two-stream mHC spans exactly the looped block [4, 8), matching the source [4, 28) cadence.
+    config["loop_settings"] = {"counts": [2], "end": 8, "eval_count": 2, "start": 4}
+    config["mhc"] = {
+        "end": 8,
+        "fixed_identity": False,
+        "iterations": 20,
+        "pre_scale": 1.0,
+        "start": 4,
+        "streams": 2,
+    }
+    config["hidden_size"] = 8
+    # Four vision blocks keep the native Qwen3.5 tower and the fixed patch/temporal/merge and position contracts.
+    config["vision_config"].update(
+        hidden_size=64,
+        intermediate_size=128,
+        num_heads=2,
+        out_hidden_size=8,
+        depth=4,
+    )
+    (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
+
+    set_seed(42)
+    model = AutoModelForImageTextToText.from_config(
+        AutoConfig.from_pretrained(folder, trust_remote_code=True),
+        trust_remote_code=True,
+        dtype=torch.bfloat16,
+    )
+    # The source keeps the hybrid linear-attention SSM parameters FP32 even under a BF16 load.
+    from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5GatedDeltaNet
+
+    for layer in model.model.language_model.layers:
+        linear = getattr(layer, "linear_attn", None)
+        if isinstance(linear, Qwen3_5GatedDeltaNet):
+            with torch.no_grad():
+                linear.A_log = torch.nn.Parameter(linear.A_log.float())
+                linear.dt_bias = torch.nn.Parameter(linear.dt_bias.float())
+                linear.norm.weight = torch.nn.Parameter(linear.norm.weight.float())
+                linear.conv1d.weight = torch.nn.Parameter(linear.conv1d.weight.float())
+    # One BF16 MTP draft layer reusing a full-attention backbone layer outside the mHC range.
+    full_layer = model.model.language_model.layers[11]
+    model.mtp = torch.nn.ModuleDict(
+        {
+            "pre_fc_norm_embedding": torch.nn.RMSNorm(8),
+            "fc": torch.nn.Linear(16, 8, bias=False),
+            "layers": torch.nn.ModuleList([copy.deepcopy(full_layer)]),
+            "norm": torch.nn.RMSNorm(8),
+            "pre_fc_norm_hidden": torch.nn.RMSNorm(8),
+        }
+    )
+    model = model.cpu()
+    representation = str(model)
+
+    state = model.state_dict()
+    backbone, draft = {}, {}
+    for name, tensor in state.items():
+        if name.startswith("mtp."):
+            draft[name] = tensor.to(torch.bfloat16).contiguous()
+        else:
+            # The shipped checkpoint stores the complete backbone FP32, mHC routers included.
+            backbone[name] = tensor.to(torch.float32).contiguous()
+    save_file(backbone, str(folder / "model.safetensors"), metadata={"format": "pt"})
+    save_file(draft, str(folder / "mtp.safetensors"), metadata={"format": "pt"})
+    # Rewrite the edited config (save was manual) and pin the source generation config bytes.
+    (folder / "config.json").write_text(json.dumps(config, indent=2) + "\n")
+    (folder / "generation_config.json").write_bytes(
+        Path(hf_hub_download(source_model_id, "generation_config.json", revision=source_revision)).read_bytes()
+    )
+
+    expected_draft = {
+        "mtp.fc.weight",
+        "mtp.pre_fc_norm_embedding.weight",
+        "mtp.pre_fc_norm_hidden.weight",
+        "mtp.norm.weight",
+        "mtp.layers.0.input_layernorm.weight",
+        "mtp.layers.0.post_attention_layernorm.weight",
+        "mtp.layers.0.self_attn.q_proj.weight",
+        "mtp.layers.0.self_attn.k_proj.weight",
+        "mtp.layers.0.self_attn.v_proj.weight",
+        "mtp.layers.0.self_attn.o_proj.weight",
+        "mtp.layers.0.self_attn.q_norm.weight",
+        "mtp.layers.0.self_attn.k_norm.weight",
+        "mtp.layers.0.mlp.gate_proj.weight",
+        "mtp.layers.0.mlp.up_proj.weight",
+        "mtp.layers.0.mlp.down_proj.weight",
+    }
+    assert set(draft) == expected_draft, sorted(set(draft) ^ expected_draft)
+    assert all(
+        name.count("mhc_attn") + name.count("mhc_ffn") == 0 or backbone[name].dtype == torch.float32
+        for name in backbone
+    )
+    return representation
+
+
+create_model(
+    source_model_id="sensenova/SenseNova-Loop-9B-Preview",
+    source_revision="9937e91c7301b7dcbb5bf17e3e4e95c499a11e99",
+    save_folder="/tmp/sensenova-loop",
+)

.meta/env.json

@@ -0,0 +1,41 @@
+{
+  "source_model": {
+    "name": "sensenova/SenseNova-Loop-9B-Preview",
+    "commit_id": "9937e91c7301b7dcbb5bf17e3e4e95c499a11e99"
+  },
+  "environments": {
+    "creation": {
+      "python": "3.12.13",
+      "packages": {
+        "huggingface_hub": "2.3.0",
+        "numpy": "2.5.2",
+        "pillow": "12.3.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "transformers": "5.22.0.dev0"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "NVIDIA H20"
+      }
+    },
+    "transformers": {
+      "python": "3.12.13",
+      "packages": {
+        "huggingface_hub": "2.3.0",
+        "numpy": "2.5.2",
+        "pillow": "12.3.0",
+        "safetensors": "0.8.0",
+        "tokenizers": "0.23.3",
+        "torch": "2.13.0+cu126",
+        "transformers": "5.22.0.dev0",
+        "triton": "3.7.1"
+      },
+      "cuda": "12.6",
+      "hardware": {
+        "gpu_model": "NVIDIA H20"
+      }
+    }
+  }
+}

.meta/model.log

@@ -0,0 +1,181 @@
+SenseNovaLoopForConditionalGeneration(
+  (model): Qwen3_5Model(
+    (visual): Qwen3_5VisionModel(
+      (patch_embed): Qwen3_5VisionPatchEmbed(
+        (proj): Conv3d(3, 64, kernel_size=(2, 16, 16), stride=(2, 16, 16))
+      )
+      (pos_embed): Embedding(2304, 64)
+      (rotary_pos_emb): Qwen3_5VisionRotaryEmbedding()
+      (blocks): ModuleList(
+        (0-3): 4 x Qwen3_5VisionBlock(
+          (norm1): LayerNorm((64,), eps=1e-06, elementwise_affine=True, bias=True)
+          (norm2): LayerNorm((64,), eps=1e-06, elementwise_affine=True, bias=True)
+          (attn): Qwen3_5VisionAttention(
+            (qkv): Linear(in_features=64, out_features=192, bias=True)
+            (proj): Linear(in_features=64, out_features=64, bias=True)
+          )
+          (mlp): Qwen3_5VisionMLP(
+            (linear_fc1): Linear(in_features=64, out_features=128, bias=True)
+            (linear_fc2): Linear(in_features=128, out_features=64, bias=True)
+            (act_fn): GELUTanh()
+          )
+        )
+      )
+      (merger): Qwen3_5VisionPatchMerger(
+        (norm): LayerNorm((64,), eps=1e-06, elementwise_affine=True, bias=True)
+        (linear_fc1): Linear(in_features=256, out_features=256, bias=True)
+        (act_fn): GELU(approximate='none')
+        (linear_fc2): Linear(in_features=256, out_features=8, bias=True)
+      )
+    )
+    (language_model): SenseNovaLoopTextModel(
+      (embed_tokens): Embedding(248320, 8, padding_idx=248044)
+      (layers): ModuleList(
+        (0-2): 3 x Qwen3_5DecoderLayer(
+          (linear_attn): Qwen3_5GatedDeltaNet(
+            (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
+            (norm): Qwen3_5RMSNormGated()
+            (out_proj): Linear(in_features=256, out_features=8, bias=False)
+            (in_proj_qkv): Linear(in_features=8, out_features=512, bias=False)
+            (in_proj_z): Linear(in_features=8, out_features=256, bias=False)
+            (in_proj_b): Linear(in_features=8, out_features=8, bias=False)
+            (in_proj_a): Linear(in_features=8, out_features=8, bias=False)
+          )
+          (mlp): Qwen3_5MLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+          (input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+        )
+        (3): Qwen3_5DecoderLayer(
+          (self_attn): Qwen3_5Attention(
+            (q_proj): Linear(in_features=8, out_features=512, bias=False)
+            (k_proj): Linear(in_features=8, out_features=128, bias=False)
+            (v_proj): Linear(in_features=8, out_features=128, bias=False)
+            (o_proj): Linear(in_features=256, out_features=8, bias=False)
+            (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          )
+          (mlp): Qwen3_5MLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+          (input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+        )
+        (4-6): 3 x MHCDecoderLayer(
+          (linear_attn): Qwen3_5GatedDeltaNet(
+            (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
+            (norm): Qwen3_5RMSNormGated()
+            (out_proj): Linear(in_features=256, out_features=8, bias=False)
+            (in_proj_qkv): Linear(in_features=8, out_features=512, bias=False)
+            (in_proj_z): Linear(in_features=8, out_features=256, bias=False)
+            (in_proj_b): Linear(in_features=8, out_features=8, bias=False)
+            (in_proj_a): Linear(in_features=8, out_features=8, bias=False)
+          )
+          (mlp): Qwen3_5MLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+          (input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (mhc_attn): MHCRouter()
+          (mhc_ffn): MHCRouter()
+        )
+        (7): MHCDecoderLayer(
+          (self_attn): Qwen3_5Attention(
+            (q_proj): Linear(in_features=8, out_features=512, bias=False)
+            (k_proj): Linear(in_features=8, out_features=128, bias=False)
+            (v_proj): Linear(in_features=8, out_features=128, bias=False)
+            (o_proj): Linear(in_features=256, out_features=8, bias=False)
+            (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          )
+          (mlp): Qwen3_5MLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+          (input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (mhc_attn): MHCRouter()
+          (mhc_ffn): MHCRouter()
+        )
+        (8-10): 3 x Qwen3_5DecoderLayer(
+          (linear_attn): Qwen3_5GatedDeltaNet(
+            (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
+            (norm): Qwen3_5RMSNormGated()
+            (out_proj): Linear(in_features=256, out_features=8, bias=False)
+            (in_proj_qkv): Linear(in_features=8, out_features=512, bias=False)
+            (in_proj_z): Linear(in_features=8, out_features=256, bias=False)
+            (in_proj_b): Linear(in_features=8, out_features=8, bias=False)
+            (in_proj_a): Linear(in_features=8, out_features=8, bias=False)
+          )
+          (mlp): Qwen3_5MLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+          (input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+        )
+        (11): Qwen3_5DecoderLayer(
+          (self_attn): Qwen3_5Attention(
+            (q_proj): Linear(in_features=8, out_features=512, bias=False)
+            (k_proj): Linear(in_features=8, out_features=128, bias=False)
+            (v_proj): Linear(in_features=8, out_features=128, bias=False)
+            (o_proj): Linear(in_features=256, out_features=8, bias=False)
+            (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+            (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          )
+          (mlp): Qwen3_5MLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+          (input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+          (post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+        )
+      )
+      (norm): Qwen3_5RMSNorm((8,), eps=1e-06)
+      (rotary_emb): Qwen3_5TextRotaryEmbedding()
+    )
+  )
+  (lm_head): Linear(in_features=8, out_features=248320, bias=False)
+  (mtp): ModuleDict(
+    (pre_fc_norm_embedding): RMSNorm((8,), eps=None, elementwise_affine=True)
+    (fc): Linear(in_features=16, out_features=8, bias=False)
+    (layers): ModuleList(
+      (0): Qwen3_5DecoderLayer(
+        (self_attn): Qwen3_5Attention(
+          (q_proj): Linear(in_features=8, out_features=512, bias=False)
+          (k_proj): Linear(in_features=8, out_features=128, bias=False)
+          (v_proj): Linear(in_features=8, out_features=128, bias=False)
+          (o_proj): Linear(in_features=256, out_features=8, bias=False)
+          (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+          (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
+        )
+        (mlp): Qwen3_5MLP(
+          (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+          (up_proj): Linear(in_features=8, out_features=32, bias=False)
+          (down_proj): Linear(in_features=32, out_features=8, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+        (post_attention_layernorm): Qwen3_5RMSNorm((8,), eps=1e-06)
+      )
+    )
+    (norm): RMSNorm((8,), eps=None, elementwise_affine=True)
+    (pre_fc_norm_hidden): RMSNorm((8,), eps=None, elementwise_affine=True)
+  )
+)

README.md

@@ -0,0 +1,107 @@
+---
+library_name: transformers
+pipeline_tag: image-text-to-text
+base_model:
+- sensenova/SenseNova-Loop-9B-Preview
+---
+
+This is a randomly initialized tiny model based on [sensenova/SenseNova-Loop-9B-Preview](https://huggingface.co/sensenova/SenseNova-Loop-9B-Preview) for testing and debugging.
+
+| Feature | Source | This repo |
+|---|---|---|
+| Checkpoint tensor elements | 243.29M BF16, 9.41B F32 | 9.19K BF16, 4.55M F32 |
+| Safetensors weight files | 38.14 GB (9 files) | 18.25 MB (2 files) |
+| Distinct / executed decoder layers | 32 / 56 (block 4-28 twice) | 12 / 16 (block 4-7 twice) |
+| Attention cadence | three linear then full, four-layer cycle | Same; four complete cycles |
+| Hidden width / MLP width | 4,096 / 12,288 | 8 / 32 |
+| Full Q heads / KV heads / head dim | 16 / 4 / 256 | 8 / 4 / 32 |
+| Linear key/value heads x head dim | 16x128 / 32x128 | 4x32 / 8x32 |
+| mHC streams / Sinkhorn iterations | 2 / 20 over layers 4-28 | 2 / 20 over layers 4-7 |
+| MTP draft layers / precision | 1 / BF16 (separate mtp.safetensors) | Same |
+| Vision depth / hidden / heads | 27 / 1,152 / 16 | 4 / 64 / 2 |
+| Patch / temporal patch / spatial merge | 16 / 2 / 2 | Same |
+| Vision position buffer | 2,304 entries | Same |
+| Vocabulary rows / tying | 248,320 / untied | Same |
+| Backbone storage precision | FP32 | Same |
+| SSM/conv/mHC parameters | FP32 | Same |
+| Maximum context length | 262,144 | Same |
+
+The pinned upstream Loop/mHC implementation is included for Transformers loading and wraps the native Qwen3.5 backbone; physical layers 4-7 share weights across both visits while every visit owns an independent dynamic-cache slot. The complete backbone (embeddings, language model, vision tower and head) is serialized in FP32 exactly as published; only the standalone MTP draft is BF16. This repo provides a Transformers example with image and video inputs; no serving-backend validation is claimed.
+
+| File path | Size |
+|------|------|
+| model.safetensors | 18.2MB |
+| mtp.safetensors | 0.0MB |
+
+### Example usage
+
+#### Transformers (✅ verified)
+
+Load the included pinned Loop/mHC implementation with `trust_remote_code=True`. The example sends one image and four predecoded video frames; the hybrid linear-attention path requires CUDA.
+
+```python
+def run_transformers(model_id: str):
+    import numpy as np
+    import torch
+    from PIL import Image
+    from transformers import AutoModelForImageTextToText, AutoProcessor
+    from transformers.video_utils import VideoMetadata
+
+    model_id = str(model_id)
+    processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
+    model = (
+        AutoModelForImageTextToText.from_pretrained(
+            model_id, trust_remote_code=True, dtype=torch.bfloat16, attn_implementation="sdpa"
+        )
+        .eval()
+        .cuda()
+    )
+    image = Image.fromarray(np.random.randint(0, 255, (256, 256, 3), dtype=np.uint8), "RGB")
+    frames = np.stack(
+        [
+            np.asarray(Image.new("RGB", (256, 256), color))
+            for color in ((220, 30, 30), (30, 220, 30), (30, 30, 220), (220, 220, 30))
+        ]
+    )
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": image},
+                {"type": "video", "video": frames},
+                {"type": "text", "text": "Describe the image and the video."},
+            ],
+        }
+    ]
+    inputs = processor.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+        enable_thinking=False,
+        processor_kwargs={"video_metadata": [VideoMetadata(total_num_frames=4, fps=2, frames_indices=[0, 1, 2, 3])]},
+    ).to(model.device)
+    assert inputs["pixel_values"].shape[0] > 0 and inputs["pixel_values_videos"].shape[0] > 0
+    with torch.no_grad():
+        logits = model(**inputs).logits
+        assert torch.isfinite(logits).all()
+        output = model.generate(**inputs, do_sample=False, min_new_tokens=8, max_new_tokens=8)
+    print(processor.decode(output[0, inputs["input_ids"].shape[1] :], skip_special_tokens=False))
+
+
+if __name__ == "__main__":
+    run_transformers(model_id="scratch/sensenova-loop")
+```
+
+### Creating this repo
+
+- Code: [.meta/create_model.py](.meta/create_model.py)
+- Model structure: [.meta/model.log](.meta/model.log)
+- Environment: [.meta/env.json](.meta/env.json)
+
+### Change log
+
+- 2026-11-03: Initial version. Twelve physical decoder layers preserve the pre-loop/loop/post-loop boundaries (layers 0-3, looped 4-7 executed twice, layers 8-11) so the repeated block is one complete four-layer linear/linear/linear/full cycle and the forward runs sixteen layer executions; two-stream mHC with 20 Sinkhorn iterations is attached to the four looped layers. The native Qwen3.5 vision tower keeps patch 16, temporal patch 2, spatial merge 2 and the 2,304-position buffer. The source stores its backbone in FP32 and its MTP draft in BF16 as a separate mtp.safetensors component; both precision contracts are preserved.
+
+*This repo is generated by GPT-6.1-Sol.*

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

config.json

@@ -0,0 +1,118 @@
+{
+  "architectures": [
+    "SenseNovaLoopForConditionalGeneration"
+  ],
+  "attn_implementation": "sdpa",
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 248046,
+  "hidden_size": 8,
+  "image_token_id": 248056,
+  "loop_settings": {
+    "counts": [
+      2
+    ],
+    "end": 8,
+    "eval_count": 2,
+    "start": 4
+  },
+  "model_type": "sensenova_loop",
+  "pad_token_id": 248044,
+  "patch_conv3d": true,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_output_gate": true,
+    "bos_token_id": null,
+    "dtype": "bfloat16",
+    "eos_token_id": 248044,
+    "full_attention_interval": 4,
+    "head_dim": 32,
+    "hidden_act": "silu",
+    "hidden_size": 8,
+    "initializer_range": 0.02,
+    "intermediate_size": 32,
+    "layer_types": [
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention",
+      "linear_attention",
+      "linear_attention",
+      "linear_attention",
+      "full_attention"
+    ],
+    "linear_conv_kernel_dim": 4,
+    "linear_key_head_dim": 32,
+    "linear_num_key_heads": 4,
+    "linear_num_value_heads": 8,
+    "linear_value_head_dim": 32,
+    "mamba_ssm_dtype": "float32",
+    "max_position_embeddings": 262144,
+    "mlp_only_layers": [],
+    "model_type": "qwen3_5_text",
+    "mtp_num_hidden_layers": 1,
+    "mtp_use_dedicated_embeddings": false,
+    "num_attention_heads": 8,
+    "num_hidden_layers": 12,
+    "num_key_value_heads": 4,
+    "pad_token_id": 248044,
+    "partial_rotary_factor": 0.25,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        1,
+        1,
+        2
+      ],
+      "partial_rotary_factor": 0.25,
+      "rope_theta": 10000000,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 248320
+  },
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "video_token_id": 248057,
+  "vision_config": {
+    "deepstack_visual_indexes": [],
+    "depth": 4,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 64,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 128,
+    "model_type": "qwen3_5_vision",
+    "num_heads": 2,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 8,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 248054,
+  "vision_start_token_id": 248053,
+  "mhc": {
+    "end": 8,
+    "fixed_identity": false,
+    "iterations": 20,
+    "pre_scale": 1.0,
+    "start": 4,
+    "streams": 2
+  },
+  "model_name": "SenseNova-Loop-9B-Preview",
+  "auto_map": {
+    "AutoConfig": "modeling_sensenova_loop.SenseNovaLoopConfig",
+    "AutoModel": "modeling_sensenova_loop.SenseNovaLoopModel",
+    "AutoModelForCausalLM": "modeling_sensenova_loop.SenseNovaLoopForConditionalGeneration",
+    "AutoModelForImageTextToText": "modeling_sensenova_loop.SenseNovaLoopForConditionalGeneration"
+  }
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [
+    248044,
+    248046
+  ],
+  "transformers_version": "5.18.0",
+  "use_cache": true,
+  "temperature": 1.0,
+  "do_sample": true,
+  "top_p": 0.95
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:063ba8b84b81edad395358bb6a51ff8e4e612a140bc62a5347b28d2aaccb845e
+size 18226832

modeling_sensenova_loop.py

@@ -0,0 +1,395 @@
+"""Hugging Face remote-code support for shared-weight SenseNova Loop and MHC.
+
+This single file contains configuration, plan validation, MHC routing and the
+model implementation. The original physical layer weights are reused on every
+Loop visit; autoregressive cache states are independent per visit.
+"""
+import copy
+import math
+from contextvars import ContextVar
+
+import torch
+from torch import nn
+from transformers import Qwen3_5Config
+from transformers.cache_utils import DynamicCache
+from transformers.models.qwen3_5 import modeling_qwen3_5 as hf
+
+
+_attention_capture = ContextVar("sensenova_attention_capture", default=None)
+
+
+class SenseNovaLoopConfig(Qwen3_5Config):
+    model_type = "sensenova_loop"
+
+
+def execution_plan(config):
+    """Return physical layer indices in the order they execute during inference."""
+    loop = config.get("loop_settings")
+    if not isinstance(loop, dict):
+        raise ValueError("This implementation requires explicit loop_settings")
+
+    text_config = config["text_config"]
+    num_layers = text_config["num_hidden_layers"]
+    start, end, count = loop.get("start"), loop.get("end"), loop.get("eval_count")
+    if any(type(value) is not int for value in (num_layers, start, end, count)) or not (
+        0 <= start < end <= num_layers and count >= 1
+    ):
+        raise ValueError("Invalid Loop interval or eval_count")
+    if loop.get("counts") != [count]:
+        raise ValueError("Only fixed-count Loop checkpoints are supported")
+
+    layer_types = text_config["layer_types"]
+    if len(layer_types) != num_layers or set(layer_types) - {"full_attention", "linear_attention"}:
+        raise ValueError("Unexpected physical layer types")
+    if "full_attention" not in layer_types:
+        raise ValueError("At least one full-attention layer is required for sequence-length tracking")
+
+    mhc = config.get("mhc")
+    if mhc is not None:
+        mhc_start, mhc_end, streams, iterations = (
+            mhc.get(key) for key in ("start", "end", "streams", "iterations")
+        )
+        if (
+            any(type(value) is not int for value in (mhc_start, mhc_end, streams, iterations))
+            or not 0 <= mhc_start < mhc_end <= num_layers
+            or streams not in (2, 4)
+            or iterations < 1
+        ):
+            raise ValueError("Invalid MHC range, stream count, or iterations")
+        if type(mhc.get("fixed_identity")) is not bool:
+            raise ValueError("MHC fixed_identity must be a boolean")
+        scale = mhc.get("pre_scale", 1.0)
+        if not isinstance(scale, (int, float)) or not math.isfinite(scale) or scale <= 0:
+            raise ValueError("Invalid MHC pre_scale")
+        allowed = {"start", "end", "streams", "iterations", "fixed_identity", "pre_scale"}
+        if set(mhc) - allowed:
+            raise ValueError("Unknown MHC configuration fields")
+        phase_boundaries = (0, start, end, num_layers)
+        if not any(
+            left <= mhc_start < mhc_end <= right
+            for left, right in zip(phase_boundaries, phase_boundaries[1:])
+        ):
+            raise ValueError("MHC cannot cross a Loop phase boundary")
+
+    return tuple(range(start)) + tuple(range(start, end)) * count + tuple(range(end, num_layers))
+
+
+def sinkhorn(logits, iterations):
+    """Normalize routing logits across rows and columns in log space."""
+    logits = logits.float()
+    for _ in range(iterations):
+        logits = logits - torch.logsumexp(logits, dim=-1, keepdim=True)
+        logits = logits - torch.logsumexp(logits, dim=-2, keepdim=True)
+    return logits.exp()
+
+
+class MHCRouter(nn.Module):
+    def __init__(self, dim, settings):
+        super().__init__()
+        self.streams = settings["streams"]
+        self.iterations = settings["iterations"]
+        self.fixed_identity = settings["fixed_identity"]
+        self.pre_scale = settings.get("pre_scale", 1.0)
+        streams = self.streams
+        output_size = 2 * streams + (0 if self.fixed_identity else streams * streams)
+        self.weight = nn.Parameter(torch.zeros(output_size, streams * dim, dtype=torch.float32))
+        self.bias = nn.Parameter(torch.zeros(output_size, dtype=torch.float32))
+        with torch.no_grad():
+            initial_probabilities = {2: (0.25, 0.75), 4: (0.0625, 0.1875, 0.25, 0.5)}[streams]
+            self.bias[:streams].copy_(torch.logit(torch.tensor(initial_probabilities, device=self.bias.device)))
+            if not self.fixed_identity:
+                diagonal = math.log((1 - (streams - 1) / 16) * 16)
+                self.bias[2 * streams:].copy_((torch.eye(streams, device=self.bias.device) * diagonal).flatten())
+
+    def _apply(self, fn, recurse=True):
+        # Keep the original FP32 values when the surrounding model is cast to BF16.
+        def keep_fp32(tensor):
+            result = fn(tensor)
+            if tensor.is_floating_point() and result.dtype != torch.float32:
+                if tensor.device.type == "meta" and result.device.type != "meta":
+                    return torch.empty_like(result, dtype=torch.float32)
+                return tensor.to(device=result.device, dtype=torch.float32)
+            return result
+
+        return super()._apply(keep_fp32, recurse=recurse)
+
+    def forward(self, residual):
+        if self.weight.dtype != torch.float32 or self.bias.dtype != torch.float32:
+            raise RuntimeError("mHC routers require FP32 parameters; configure FSDP per-module precision")
+        with torch.autocast(device_type=residual.device.type, enabled=False):
+            flat = residual.flatten(-2).float()
+            normalized = flat * torch.rsqrt(flat.square().mean(-1, keepdim=True) + 1e-6)
+            logits = torch.nn.functional.linear(normalized, self.weight, self.bias)
+            streams = self.streams
+            pre = logits[..., :streams].sigmoid()
+            post = 2 * logits[..., streams:2 * streams].sigmoid()
+            if self.fixed_identity:
+                mix = torch.eye(streams, device=flat.device, dtype=torch.float32).expand(
+                    *flat.shape[:-1], streams, streams
+                )
+            else:
+                mix = sinkhorn(logits[..., 2 * streams:].unflatten(-1, (streams, streams)), self.iterations)
+            branch = torch.matmul(pre.to(residual.dtype).unsqueeze(-2), residual).squeeze(-2)
+            if self.pre_scale != 1.0:
+                branch = branch * self.pre_scale
+        return branch, post, mix
+
+    @staticmethod
+    def merge(residual, branch, post, mix):
+        """Mix residual streams and add the branch output to each stream."""
+        with torch.autocast(device_type=residual.device.type, enabled=False):
+            mixed_residual = torch.matmul(mix.to(residual.dtype), residual)
+            branch_update = post.to(branch.dtype).unsqueeze(-1) * branch.unsqueeze(-2)
+            return mixed_residual + branch_update
+
+
+def enter(hidden, physical, settings):
+    """Expand to residual streams at the first mHC layer, or restore their shape."""
+    if physical == settings["start"]:
+        return hidden.unsqueeze(-2).expand(
+            *hidden.shape[:-1], settings["streams"], hidden.shape[-1]
+        ).contiguous()
+    return hidden.unflatten(-1, (settings["streams"], -1))
+
+
+def leave(residual, physical, settings):
+    """Reduce streams at the last mHC layer; otherwise carry them forward."""
+    return residual.mean(-2) if physical == settings["end"] - 1 else residual.flatten(-2)
+
+
+class _CacheLayersView:
+    """Expose the execution slot when a layer accesses its physical index."""
+    def __init__(self, cache, physical, virtual):
+        self.cache = cache
+        self.physical = physical
+        self.virtual = virtual
+
+    def __getitem__(self, index):
+        if index != self.physical:
+            raise IndexError("A physical layer may only access its own virtual cache slot")
+        return self.cache.layers[self.virtual]
+
+
+class _VisitCache:
+    """Map a physical attention module's layer_idx to one execution slot.
+
+    Mapping is local to this forward call: shared modules and their layer_idx are
+    never mutated. `layers` supports Transformers 5.12; state accessors support
+    newer official backbone cache APIs as well.
+    """
+    def __init__(self, cache, physical, virtual):
+        self.cache = cache
+        self.virtual = virtual
+        self.layers = _CacheLayersView(cache, physical, virtual)
+
+    def update(self, key_states, value_states, layer_idx, **kwargs):
+        return self.cache.update(key_states, value_states, self.virtual, **kwargs)
+
+    def has_previous_state(self, layer_idx=None, **kwargs):
+        return self.cache.has_previous_state(self.virtual, **kwargs)
+
+    def update_conv_state(self, state, layer_idx, **kwargs):
+        return self.cache.update_conv_state(state, self.virtual, **kwargs)
+
+    def update_recurrent_state(self, state, layer_idx, **kwargs):
+        return self.cache.update_recurrent_state(state, self.virtual, **kwargs)
+
+    def get_conv_state(self, layer_idx, **kwargs):
+        return self.cache.get_conv_state(self.virtual, **kwargs)
+
+    def get_recurrent_state(self, layer_idx, **kwargs):
+        return self.cache.get_recurrent_state(self.virtual, **kwargs)
+
+    def is_recording_past(self, layer_idx):
+        return self.cache.is_recording_past(self.virtual)
+
+
+class SenseNovaLoopTextModel(hf.Qwen3_5TextModel):
+    def forward(self, input_ids=None, attention_mask=None, position_ids=None,
+                past_key_values=None, inputs_embeds=None, use_cache=None,
+                output_hidden_states=None, output_attentions=None, **kwargs):
+        """Run Loop visits, optionally returning their states and attention weights.
+
+        Hidden states include the initial embedding and one state per visit.
+        MHC states between its boundaries retain their flattened residual streams.
+        Attention entries follow visit order, with None for linear-attention layers.
+        """
+        if (input_ids is None) == (inputs_embeds is None):
+            raise ValueError("Specify exactly one of input_ids or inputs_embeds")
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+        use_cache = self.config.use_cache if use_cache is None else use_cache
+        if output_hidden_states is None:
+            output_hidden_states = getattr(self.config, "output_hidden_states", False)
+        if output_attentions is None:
+            output_attentions = getattr(self.config, "output_attentions", False)
+        if not isinstance(output_hidden_states, bool):
+            raise TypeError("output_hidden_states must be a boolean.")
+        if output_attentions and self.config._attn_implementation != "eager":
+            raise ValueError(
+                "Attention weights require attn_implementation='eager' when loading the model."
+            )
+        if self.gradient_checkpointing and self.training and use_cache:
+            raise ValueError("Disable use_cache when training with gradient checkpointing")
+        if not use_cache and past_key_values is not None:
+            raise ValueError("past_key_values requires use_cache=True")
+        if past_key_values is not None and not isinstance(past_key_values, DynamicCache):
+            raise TypeError("Loop generation currently supports DynamicCache only")
+        if use_cache:
+            if past_key_values is None:
+                past_key_values = DynamicCache(config=self._loop_cache_config)
+                past_key_values._loop_execution_plan = self._loop_plan
+            elif getattr(past_key_values, "_loop_execution_plan", None) != self._loop_plan:
+                # GenerationMixin may allocate an empty physical-depth cache.
+                # A populated foreign cache cannot be reinterpreted safely.
+                if past_key_values.get_seq_length() != 0 or past_key_values.has_previous_state():
+                    raise ValueError("A populated cache from another execution plan cannot be reused")
+                past_key_values = DynamicCache(config=self._loop_cache_config)
+                past_key_values._loop_execution_plan = self._loop_plan
+
+        if position_ids is None:
+            seen = past_key_values.get_seq_length() if past_key_values is not None else 0
+            position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + seen
+            position_ids = position_ids.view(1, 1, -1).expand(4, inputs_embeds.shape[0], -1)
+        elif position_ids.ndim == 2:
+            position_ids = position_ids.unsqueeze(0).expand(4, -1, -1)
+        if position_ids.ndim == 3 and position_ids.shape[0] == 4:
+            text_positions, position_ids = position_ids[0], position_ids[1:]
+        else:
+            text_positions = None
+
+        if isinstance(attention_mask, dict):
+            masks = attention_mask
+        else:
+            mask_args = dict(
+                config=self._loop_cache_config,
+                inputs_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                past_key_values=past_key_values,
+                position_ids=text_positions,
+            )
+            if hasattr(hf, "create_recurrent_attention_mask"):
+                linear_mask = hf.create_recurrent_attention_mask(**mask_args)
+            else:
+                linear_mask = self._update_linear_attn_mask(attention_mask, past_key_values)
+            masks = {
+                "full_attention": hf.create_causal_mask(**mask_args),
+                "linear_attention": linear_mask,
+            }
+
+        hidden = inputs_embeds
+        position_embeddings = self.rotary_emb(hidden, position_ids)
+        hidden_states = [hidden] if output_hidden_states else None
+        attentions = [] if output_attentions else None
+        hooks = []
+        capture_token = None
+
+        def record_attention(_module, _inputs, output):
+            if _attention_capture.get() is attentions:
+                if output[1] is None:
+                    raise RuntimeError("The attention backend did not return attention weights.")
+                attentions.append(output[1])
+
+        # Repeated visits share weights, but each visit has its own cache slot.
+        try:
+            if attentions is not None:
+                for layer in self.layers:
+                    if hasattr(layer, "self_attn"):
+                        hooks.append(layer.self_attn.register_forward_hook(record_attention))
+                capture_token = _attention_capture.set(attentions)
+            for execution_index, physical_index in enumerate(self._loop_plan):
+                layer_type = self.config.layer_types[physical_index]
+                if attentions is not None and layer_type == "linear_attention":
+                    attentions.append(None)
+                cache = (
+                    _VisitCache(past_key_values, physical_index, execution_index)
+                    if use_cache else None
+                )
+                hidden = self.layers[physical_index](
+                    hidden, position_embeddings=position_embeddings,
+                    attention_mask=masks[layer_type],
+                    position_ids=text_positions, past_key_values=cache,
+                    use_cache=use_cache, **kwargs,
+                )
+                if hidden_states is not None:
+                    hidden_states.append(hidden)
+        finally:
+            if capture_token is not None:
+                _attention_capture.reset(capture_token)
+            for hook in hooks:
+                hook.remove()
+
+        hidden = self.norm(hidden)
+        if hidden_states is not None:
+            hidden_states[-1] = hidden
+        return hf.Qwen3_5ModelOutputWithPast(
+            last_hidden_state=hidden, past_key_values=past_key_values,
+            hidden_states=tuple(hidden_states) if hidden_states is not None else None,
+            attentions=tuple(attentions) if attentions is not None else None,
+        )
+
+
+class MHCDecoderLayer(hf.Qwen3_5DecoderLayer):
+    def forward(self, hidden_states, position_embeddings, attention_mask=None,
+                position_ids=None, past_key_values=None, **kwargs):
+        residual = enter(hidden_states, self._mhc_physical, self._mhc_settings)
+        branch, post, mix = self.mhc_attn(residual)
+        branch = self.input_layernorm(branch)
+        kind = self.layer_type if hasattr(self, "layer_type") else self.block_type
+        if kind == "linear_attention":
+            branch = self.linear_attn(hidden_states=branch, cache_params=past_key_values,
+                                      attention_mask=attention_mask, **kwargs)
+        else:
+            branch, _ = self.self_attn(
+                hidden_states=branch, position_embeddings=position_embeddings,
+                attention_mask=attention_mask, position_ids=position_ids,
+                past_key_values=past_key_values, **kwargs)
+        residual = MHCRouter.merge(residual, branch, post, mix)
+        branch, post, mix = self.mhc_ffn(residual)
+        branch = self.mlp(self.post_attention_layernorm(branch))
+        return leave(MHCRouter.merge(residual, branch, post, mix),
+                     self._mhc_physical, self._mhc_settings)
+
+
+def _configure_text_model(text, config):
+    raw = config.to_dict()
+    plan = execution_plan(raw)
+    # Change only these new model instances, preserving canonical state_dict
+    # names and avoiding a second allocation of the full backbone.
+    text.__class__ = SenseNovaLoopTextModel
+    text._loop_plan = plan
+    text._loop_cache_config = copy.deepcopy(text.config)
+    text._loop_cache_config.num_hidden_layers = len(plan)
+    text._loop_cache_config.layer_types = [text.config.layer_types[i] for i in plan]
+    if raw.get("mhc") is not None:
+        settings = raw["mhc"]
+        for i in range(settings["start"], settings["end"]):
+            layer = text.layers[i]
+            layer.__class__ = MHCDecoderLayer
+            layer._mhc_physical, layer._mhc_settings = i, dict(settings)
+            layer.mhc_attn = MHCRouter(text.config.hidden_size, settings)
+            layer.mhc_ffn = MHCRouter(text.config.hidden_size, settings)
+
+
+class SenseNovaLoopModel(hf.Qwen3_5Model):
+    config_class = SenseNovaLoopConfig
+    _keep_in_fp32_modules = ["mhc_attn", "mhc_ffn"]
+    _no_split_modules = ["Qwen3_5DecoderLayer", "MHCDecoderLayer", "Qwen3_5VisionBlock"]
+    _keys_to_ignore_on_load_unexpected = [r"^mtp\."]
+
+    def __init__(self, config):
+        execution_plan(config.to_dict())
+        super().__init__(config)
+        _configure_text_model(self.language_model, config)
+
+
+class SenseNovaLoopForConditionalGeneration(hf.Qwen3_5ForConditionalGeneration):
+    config_class = SenseNovaLoopConfig
+    _keep_in_fp32_modules = ["mhc_attn", "mhc_ffn"]
+    _no_split_modules = ["Qwen3_5DecoderLayer", "MHCDecoderLayer", "Qwen3_5VisionBlock"]
+    _keys_to_ignore_on_load_unexpected = [r"^mtp\."]
+
+    def __init__(self, config):
+        execution_plan(config.to_dict())
+        super().__init__(config)
+        _configure_text_model(self.model.language_model, config)

mtp.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8ba6e24e21805901db005ca4e21c3e9e5364550439a3883f797cf183332d5ed
+size 19824

preprocessor_config.json

@@ -0,0 +1,26 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "merge_size": 2,
+  "patch_size": 16,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 16777216,
+    "shortest_edge": 65536
+  },
+  "temporal_patch_size": 2
+}

processor_config.json

@@ -0,0 +1,60 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "merge_size": 2,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 16777216,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "Qwen3VLProcessor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen3VLVideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
+size 19989325

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "Qwen3VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

video_preprocessor_config.json

@@ -0,0 +1,31 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "do_sample_frames": true,
+  "fps": 2,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "max_frames": 768,
+  "merge_size": 2,
+  "min_frames": 4,
+  "patch_size": 16,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "return_metadata": false,
+  "size": {
+    "longest_edge": 25165824,
+    "shortest_edge": 4096
+  },
+  "temporal_patch_size": 2,
+  "video_processor_type": "Qwen3VLVideoProcessor"
+}