#!/usr/bin/env python3 """Single source of truth for evaluation prompts. Exactly ONE prompt per task — the one used by ``eval/task/eval.sh`` — kept byte-for-byte aligned with the SFT training data in ``data/joint/sft_joint_all.jsonl``. Task -> prompt mapping (verbatim from sft_joint_all.jsonl): video_qa_mc (videomme, videommmu, mmvu, mvbench, videoholmes, longvideobench, lvbench, mlvu) "{question}\\nOptions:\\n{opts}\\n" + VIDEO_QA_MC_TAIL spatial_intelligence (vsi) MC -> "{q}\\nOptions:\\n{opts}\\n" + VSI_MC_TAIL integer -> "{q} " + VSI_INTEGER_TAIL meters -> "{q} " + VSI_METERS_TAIL cm -> "{q} " + VSI_CENTIMETERS_TAIL room m2 -> "{q} " + VSI_SQUARE_METERS_TAIL (each prefixed with "These are frames of a video.\\n") sensenova_si / image_sequence_mc (mmsi, mindcube) "{q}\\n" + SENSENOVA_MC_TAIL (alias PROMPT_TAIL) spatial grounding (spatial_grounding) build_spatial_grounding_prompt(expr) == QWEN_NATIVE_PROMPT_SG tracking GROUNDING_QUESTION_TEMPLATE_NO_THINK.format(Question=q) + TRACKING_TAIL stvg GROUNDING_QUESTION_TEMPLATE_NO_THINK.format( Question=TRAIN_STVG_QUESTION_PREFIX.format(query=q)) + STVG_TAIL segmentation "{question}\\n" + TRAIN_SEG_IMAGE_TAIL / TRAIN_SEG_VIDEO_TAIL temporal grounding (temporal_grounding) TEMPORAL_GROUNDING_PROMPT.format(event) Keep this file dependency-free (stdlib only) so it can be imported from every worker regardless of the runtime environment. """ # =========================================================================== # Multiple-choice / numeric answer tails # =========================================================================== # video_qa_mc — videomme, videommmu, mmvu, mvbench, videoholmes, # longvideobench, lvbench, mlvu. VIDEO_QA_MC_TAIL = ( "Answer with the option letter only within ... tags. " "Example: A" ) # spatial_intelligence multiple choice — vsi (object_rel_distance, # object_rel_direction, obj_appearance_order, route_plan, ...). VSI_MC_TAIL = ( "Answer with the option letter within ... tags. " "Example: A" ) # spatial_intelligence numeric — vsi. VSI_INTEGER_TAIL = ( "Answer with an integer within ... tags. Example: 3" ) VSI_METERS_TAIL = ( "Answer with a number in meters within ... tags. Example: 2.3" ) VSI_CENTIMETERS_TAIL = ( "Answer with a number in centimeters within ... tags. Example: 120.5" ) VSI_SQUARE_METERS_TAIL = ( "Answer with a number in square meters within ... tags. Example: 25.5" ) # sensenova_si / image_sequence_mc_answer_only — mmsi, mindcube. SENSENOVA_MC_TAIL = ( "Choose the best answer from the options. " "Put exactly one uppercase option letter inside ... " "Do not explain. Example: A" ) # Backwards-compatible alias used throughout eval_vllm.py. PROMPT_TAIL = SENSENOVA_MC_TAIL # =========================================================================== # Tracking # =========================================================================== # Question scaffold shared by tracking + stvg (the tail carries the full # answer-format spec). GROUNDING_QUESTION_TEMPLATE_NO_THINK = ( "{Question}\n" "Please answer this question based on the visual content. " ) TRACKING_TAIL = ( "Please track the target object throughout the video and provide one bounding box per second, " "ONLY up to 32 seconds, within the ... tags.\n" "Example:\n" "{\"boxes\": {\"1\": [405, 230, 654, 463], \"2\": [435, 223, 678, 446], " "\"32\": [415, 203, 691, 487]}}\n" "Note: Each key in 'boxes' must correspond to a second (1, 2, 3, ..., 32) " "and contain a 4-number bounding box [x1, y1, x2, y2]." ) # =========================================================================== # Spatial-temporal grounding (STVG) # =========================================================================== STVG_TAIL = ( "Please provide only the time span in seconds and bounding boxes as JSON " "within the ... tags.\n" "You MUST output one bounding box for every integer second within the " "given time span (inclusive).\n" "Example:\n" "{\"time\": [8.1, 13.5], \"boxes\": {\"9\": [317, 422, 582, 997], " "\"10\": [332, 175, 442, 369], \"11\": [340, 180, 450, 370]}}\n" "Note: Each key in 'boxes' must be an integer second within the span, " "and its value must be a 4-number bounding box [x1, y1, x2, y2]." ) # Rewrites an eval question into the exact wording used during STVG training. TRAIN_STVG_QUESTION_PREFIX = ( 'Given the query "{query}", when and where does the described content ' 'occur in the video? please firstly give the start and end time, spatial ' 'bounding box corresponding to each integer second.' ) # =========================================================================== # Segmentation # =========================================================================== # Matches sft_joint_all.jsonl seg_image / seg_video samples. TRAIN_SEG_IMAGE_TAIL = ( "Please answer this question based on the visual content. " "This task prepares inputs for image object segmentation with a specialized model (e.g., SAM2).\n" "Please provide ONE bounding box, 3 positive points (clearly INSIDE the object), " "and 3 negative points (clearly OUTSIDE the object) within the ... tags.\n" "Choose informative points that help distinguish object vs. background. Prefer negatives on clear non-object " "pixels INSIDE the box when safe; otherwise place them just outside on obvious background. " "Negatives must NEVER be on the object or on its boundary.\n" "Example: {\"boxes\": [x1, y1, x2, y2], \"positive_points\": [[x,y],[x,y],[x,y]], " "\"negative_points\": [[x,y],[x,y],[x,y]]}" ) TRAIN_SEG_VIDEO_TAIL = ( "Please answer this question based on the visual content. " "This task prepares inputs for video object segmentation with a specialized model (e.g., SAM2).\n" "Please select ONE representative time (in seconds), and provide ONE bounding box, " "3 positive points (clearly INSIDE the object), and 3 negative points (clearly OUTSIDE the object) " "within the ... tags.\n" "Choose informative points that help distinguish object vs. background. Prefer negatives on clear non-object " "pixels INSIDE the box when safe; otherwise place them just outside on obvious background. " "Negatives must NEVER be on the object or on its boundary.\n" "Example: {\"time\": , \"boxes\": [x1, y1, x2, y2], " "\"positive_points\": [[x,y],[x,y],[x,y]], \"negative_points\": [[x,y],[x,y],[x,y]]}" ) # =========================================================================== # Spatial grounding (RefCOCO / spatial_grounding) # =========================================================================== # Matches sft_joint_all.jsonl spatial grounding samples after the token. QWEN_NATIVE_PROMPT_SG = ( 'Locate "{}" in the image. Output its bounding box in JSON format ' 'within ... tags. ' 'Example: [{{"bbox_2d": [123, 30, 404, 846]}}]' ) # =========================================================================== # Temporal grounding (TimeLens / temporal_grounding) # =========================================================================== # Matches sft_joint_all.jsonl temporal grounding samples (after