fix chunking error

processing_qwen3_vl.py

@@ -220,16 +220,17 @@ class ZFQwen3VLProcessor(ProcessorMixin):
         return_mm_token_type_ids = output_kwargs["text_kwargs"].pop("return_mm_token_type_ids", None)
         text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) # type: ignore
         self._check_special_mm_tokens(text, text_inputs, modalities=["image", "video"]) # type: ignore
+        array_ids = np.array(text_inputs["input_ids"])
+        array_attention_mask = np.array(text_inputs["attention_mask"]) if "attention_mask" in text_inputs else None
 
         if return_mm_token_type_ids:
-            array_ids = np.array(text_inputs["input_ids"])
             mm_token_type_ids = np.zeros_like(text_inputs["input_ids"])
             mm_token_type_ids[array_ids == self.image_token_id] = 1
             text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist()
 
         chunks = chunk_tokens(
             max_chunk_size=self.video_processor.max_chunk_size, # type: ignore
-            input_ids=np.array(text_inputs["input_ids"]),
+            input_ids=array_ids,
             image_token_id=self.image_token_id,
             video_token_id=self.video_token_id,
             merge_size=self.image_processor.merge_size, # type: ignore
@@ -237,14 +238,20 @@ class ZFQwen3VLProcessor(ProcessorMixin):
             image_grid_thw=image_grid_thw,
             video_grid_thw=video_grid_thw,
         )
-        image_token_mask = (text_inputs["input_ids"] == self.image_token_id)
-        video_token_mask = (text_inputs["input_ids"] == self.video_token_id)
+        image_token_mask = (array_ids == self.image_token_id).astype(int)
+        video_token_mask = (array_ids == self.video_token_id).astype(int)
+        text_token_mask = np.ones_like(image_token_mask) - image_token_mask - video_token_mask
+        if array_attention_mask is not None:
+            text_token_mask = text_token_mask * array_attention_mask
+            image_token_mask = image_token_mask * array_attention_mask
+            video_token_mask = video_token_mask * array_attention_mask
 
         return BatchFeature(data={
             **text_inputs,
             **image_inputs,
             **videos_inputs,
             "token_chunks": chunks,
+            "text_token_mask": text_token_mask,
             "image_token_mask": image_token_mask,
             "video_token_mask": video_token_mask,
         }, tensor_type=return_tensors)