Fix transformers 5.9+ compatibility: create_causal_mask kwargs

modeling.py

@@ -1,3 +1,4 @@
+import inspect
 from typing import Callable, Literal
 import numpy as np
 import torch
@@ -11,6 +12,14 @@ from .configuration import PPLXQwen3Config
 from transformers import AutoTokenizer
 from .st_quantize import FlexibleQuantizer
 
+# The transformers `create_causal_mask` signature has shifted over releases
+# (the embeds kwarg was renamed `input_embeds` -> `inputs_embeds`, and
+# `cache_position` was eventually dropped). Probe the actual signature at import
+# time so this works on any installed release, including dev/main builds.
+_CCM_PARAMS = inspect.signature(create_causal_mask).parameters
+_CCM_EMBEDS_KEY = "inputs_embeds" if "inputs_embeds" in _CCM_PARAMS else "input_embeds"
+_CCM_ACCEPTS_CACHE_POSITION = "cache_position" in _CCM_PARAMS
+
 
 # From modeling_t5gemma.py
 def bidirectional_mask_function(attention_mask: torch.Tensor | None) -> Callable:
@@ -57,21 +66,19 @@ class PPLXQwen3Model(Qwen3Model):
             inputs_embeds = self.embed_tokens(input_ids)
             input_ids = None
 
-        # We construct a dummy tensor imitating initial positions
-        dummy_cache_position = torch.arange(
-            inputs_embeds.shape[1], device=inputs_embeds.device, dtype=torch.long
-        )
-        attention_mask = {
-            "full_attention": create_causal_mask(
-                config=self.config,
-                input_embeds=inputs_embeds,
-                attention_mask=attention_mask,
-                cache_position=dummy_cache_position,
-                past_key_values=None,
-                position_ids=position_ids,
-                or_mask_function=bidirectional_mask_function(attention_mask),
-            )
+        mask_kwargs = {
+            "config": self.config,
+            _CCM_EMBEDS_KEY: inputs_embeds,
+            "attention_mask": attention_mask,
+            "past_key_values": None,
+            "position_ids": position_ids,
+            "or_mask_function": bidirectional_mask_function(attention_mask),
         }
+        if _CCM_ACCEPTS_CACHE_POSITION:
+            mask_kwargs["cache_position"] = torch.arange(
+                inputs_embeds.shape[1], device=inputs_embeds.device, dtype=torch.long
+            )
+        attention_mask = {"full_attention": create_causal_mask(**mask_kwargs)}
 
         outputs = super().forward(
             input_ids=input_ids,