import argparse
import torch
from transformers import AutoProcessor # Assuming you use HuggingFace for model/processor loading
import os
import re
from src.vllm_inference.vllm_infer import vllmWrapper # Core inference logic
from src.vllm_inference.utils import _read_video_decord_w_timestamp, monkey_patch # Video processing
from src.utils.vision_process import smart_nframes # Video processing helper
from src.utils import process_vision_info_v3
import time
import json
# Apply monkey patch for video reading if necessary
monkey_patch()
PROMPT_TEMPLATE = """
To accurately pinpoint the event "{}" in the video, determine the precise time period of the event.
Output your thought process within the tags, including analysis with either specific time ranges (xx.xx to xx.xx) in tags.
Then, provide the start and end times (in seconds, precise to two decimal places) in the format "start time to end time" within the tags. For example: "12.54 to 17.83".
"""
def get_args():
parser = argparse.ArgumentParser(
description="Evaluation for training-free video temporal grounding (Single GPU Version)"
)
parser.add_argument(
"--model_base", type=str, default="./ckpts/Time-R1-7B"
)
parser.add_argument("--batch_size", type=int, default=1, help="Batch size")
parser.add_argument(
"--output_dir",
type=str,
default="logs/demo",
help="Directory to save checkpoints",
)
parser.add_argument(
"--device", type=str, default="cuda:0", help="GPU device to use"
)
parser.add_argument(
"--pipeline_parallel_size", type=int, default=1, help="GPU nodes"
)
parser.add_argument(
"--video_path", type=str, default="./assets/OHOFG.mp4"
)
parser.add_argument(
"--query", type=str, default="person sitting down in a chair."
)
parser.add_argument("--max_new_tokens", type=int, default=128)
parser.add_argument(
"--total_pixels", type=int, default=3584 * 28 * 28, help="total_pixels"
)
return parser.parse_args()
def preprocess(processor, itm, ele):
if "video_start" in itm and itm["video_start"] is not None:
ele["video_start"] = itm["video_start"]
if "video_end" in itm and itm["video_end"] is not None:
ele["video_end"] = itm["video_end"]
messages = [
{"role": "system", "content": []},
{"role": "user", "content": []},
]
messages[0]["content"].append({"type": "text", "text": "You are a helpful assistant."})
messages[1]["content"].append({"type": "video", "video": itm["video"], **ele})
messages[1]["content"].append(
{
"type": "text",
"text": PROMPT_TEMPLATE.format(itm["sentence"]),
}
)
_, video_inputs, utils = process_vision_info_v3(
messages, return_video_kwargs=True
)
text = processor.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
return {"text": text, "videos": video_inputs, "fps": utils["fps"]}
def build_dataset(
data,
processor,
num_workers=8,
sys_prompt="You are a helpful assistant.",
min_pixels=16 * 28 * 28,
total_pixels=3584 * 28 * 28,
use_huggingface=False,
):
kwargs = {
"min_pixels": min_pixels,
"total_pixels": total_pixels,
"sys_prompt": sys_prompt,
}
ele = {
"min_pixels": min_pixels,
"total_pixels": total_pixels,
}
inputs = preprocess(processor, data, ele)
multi_modal_data = {}
if "images" in inputs and inputs["images"] is not None:
multi_modal_data["image"] = inputs["images"]
if "videos" in inputs and inputs["videos"] is not None:
multi_modal_data["video"] = inputs["videos"]
return {
"inputs": {
"raw_prompt_ids": [processor.tokenizer.encode(
inputs["text"], add_special_tokens=False
)],
"multi_modal_data": [multi_modal_data],
"mm_processor_kwargs": [(
{"fps": inputs["fps"]} if inputs["fps"] is not None else {}
)],
},
"timestamps": [data["timestamp"]],
"duration": [data["duration"]],
"video_paths": [data["video"]],
}
def extract_answer(output_string):
matches = re.findall(r"(\d+\.?\d*) (to|and) (\d+\.?\d*)", output_string)
if not matches:
answer_match = re.search(r"(.*?)", output_string)
if answer_match:
answer_content = answer_match.group(1).strip()
answer_matches = re.findall(
r"(\d+\.?\d*) (to|and) (\d+\.?\d*)", answer_content
)
if answer_matches:
last_match = answer_matches[-1]
return [float(last_match[0]), float(last_match[2])]
return [None, None]
last_match = matches[-1]
start_time_str = last_match[0]
end_time_str = last_match[2]
try:
start_time = float(start_time_str)
end_time = float(end_time_str)
return [start_time, end_time]
except ValueError:
return [None, None]
def main(args):
args = get_args()
os.makedirs(args.output_dir, exist_ok=True)
output_file = os.path.join(
args.output_dir, f"tmp_output.jsonl"
)
# build model and processor
processor = AutoProcessor.from_pretrained(args.model_base, use_fast=True)
processor.tokenizer.padding_side = "left"
model = vllmWrapper(args)
data = {
"video": args.video_path,
"duration": 35.04, # Duration of the video in seconds, read the whole video
"timestamp": [ # GT timestamps
1.0,
7.5
],
"sentence": args.query,
}
data_args = {
"num_workers": min(8, args.batch_size),
"total_pixels": args.total_pixels,
}
data = build_dataset(data, processor, **data_args)
program_start_time = time.perf_counter()
output_texts = model.generate(
data["inputs"],
max_new_tokens=args.max_new_tokens,
)
targets = data["timestamps"]
f = open(output_file, "a+")
for i in range(len(targets)):
pred = extract_answer(output_texts[i])
print(output_texts[i], pred)
f.write(
json.dumps(
{
"pred": pred,
"target": list(targets[i]),
"duration": (
None
if "duration" not in data
else data["duration"][i]
),
"output_text": output_texts[i],
}
)
+ "\n"
)
f.flush()
# --- END TOTAL TIME & CALCULATIONS ---
program_end_time = time.perf_counter()
total_program_duration = program_end_time - program_start_time
print("\n--- Timing Summary ---")
print(f"Total program execution time: {total_program_duration:.2f} seconds")
output_filename = f"{args.output_dir}/timing_summary_vllm.txt"
with open(output_filename, "w", encoding="utf-8") as f:
f.write("\n--- Timing Summary ---\n")
f.write(f"Total program execution time: {total_program_duration:.2f} seconds\n")
f.write("Another line of summary using write.\n")
if __name__ == "__main__":
from src.vllm_inference.utils import monkey_patch
monkey_patch()
args = get_args()
main(args)