Use the repo's reference transformers inference path (apply_chat_template tokenize=True)

app.py

@@ -25,7 +25,6 @@ import gradio as gr  # noqa: E402
 import imageio.v2 as imageio  # noqa: E402
 import numpy as np  # noqa: E402
 import torch  # noqa: E402
-from molmo_utils import process_vision_info  # noqa: E402
 from transformers import AutoModelForImageTextToText, AutoProcessor  # noqa: E402
 
 MODEL_ID = "tidalove/Molmo2Fish"
@@ -38,6 +37,7 @@ SAMPLING_FPS = 2
 
 TRACK_STYLE = "video_point_track_per_frame"
 DEFAULT_PROMPT = "track all fish"
+IM_END_TOKEN_ID = 151937  # <im_end>; config.image_end_token_id
 
 # html-v2 pointing format, exactly as in olmo/preprocessing/point_formatter.py
 COORD_RE = re.compile(r"<(?:points|tracks).*? coords=\"([0-9\t:;, .]+)\"/?>")
@@ -167,14 +167,10 @@ def build_messages(video_path: str, turns: list) -> list:
     for i, (user_text, assistant_text) in enumerate(turns):
         content = [dict(type="text", text=user_text, style=TRACK_STYLE)]
         if i == 0:
-            content.append(dict(
-                type="video",
-                video=video_path,
-                num_frames=NUM_FRAMES,
-                frame_sample_mode=FRAME_SAMPLE_MODE,
-                max_fps=MAX_FPS,
-                sampling_fps=SAMPLING_FPS,
-            ))
+            # Frame sampling (num_frames=128, uniform_last_frame, max_fps/sampling_fps=2)
+            # comes from the model's own video_preprocessor_config.json, so the path is
+            # all the processor needs — same as olmo/hf_model/test_molmo2.py.
+            content.append(dict(type="video", video=video_path))
         messages.append({"role": "user", "content": content})
         if assistant_text is not None:
             messages.append({"role": "assistant",
@@ -184,18 +180,21 @@ def build_messages(video_path: str, turns: list) -> list:
 
 def run_model(video_path: str, turns: list, max_new_tokens: int) -> str:
     messages = build_messages(video_path, turns)
-    _, videos, video_kwargs = process_vision_info(messages)
-    frames, metadatas = zip(*videos)
-    text = processor.apply_chat_template(
-        messages, tokenize=False, add_generation_prompt=True
-    )
-    inputs = processor(
-        videos=list(frames),
-        video_metadata=list(metadatas),
-        text=text,
-        padding=True,
+    # Reference path from the repo's own olmo/hf_model/test_molmo2.py: let the
+    # Molmo2Processor decode + sample the video and expand <|video|> itself.
+    inputs = processor.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
         return_tensors="pt",
-        **video_kwargs,
+        return_dict=True,
+    )
+    ids = inputs["input_ids"]
+    print(
+        f"[molmo2fish] input_ids={tuple(ids.shape)} "
+        f"im_end={int((ids == IM_END_TOKEN_ID).sum())} "
+        f"keys={sorted(inputs.keys())}",
+        flush=True,
     )
     inputs = {k: (v.to(model.device) if hasattr(v, "to") else v)
               for k, v in inputs.items()}