#!/usr/bin/env python3 """Shot Reader: read one video shot and write its shot sheet (camera, subjects, paths, events, set) as JSON. python shot_reader.py clip.mp4 --subject "P01: person (woman in a black polo)" --subject "O01: dog" python shot_reader.py clip.mp4 --subject "P01: person" --backend vllm # fastest python shot_reader.py clip.mp4 --subject "P01: person" --load-in-4bit # ~6 GB of VRAM One shot = one continuous take (cut longer videos into shots first). The model sees 16 frames sampled evenly across the shot at 448x256, plus the ids and classes of the subjects you name; it keeps your ids. Needs: torch, transformers>=4.57, pillow, ffmpeg on PATH (+ vllm or bitsandbytes for those options). """ import argparse, json, os, re, subprocess, sys, tempfile HERE = os.path.dirname(os.path.abspath(__file__)) REPO = "voyager-so/shot-reader-8b" N_FRAMES, SIZE = 16, (448, 256) PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s (frame i is at t = {times}). Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward). Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f} (add intermediate keys when it moves). Keep the given ids. Subjects in the brief (id: class): {subjects} {schema} Reply with only the JSON object, no prose, no code fences.""" def schema_doc(model): p = os.path.join(HERE, "schema.md") if not os.path.exists(p): from huggingface_hub import hf_hub_download p = hf_hub_download(model if "/" in model and not os.path.isdir(model) else REPO, "schema.md") return open(p).read() def duration(path): out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path], capture_output=True, text=True, check=True).stdout return float(out.strip()) def sample_frames(path, dur): from PIL import Image times = [i * dur / N_FRAMES for i in range(N_FRAMES)] imgs = [] with tempfile.TemporaryDirectory() as d: for i, t in enumerate(times): f = os.path.join(d, f"{i}.jpg") subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path, "-frames:v", "1", "-q:v", "2", f], check=True) imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC)) return times, imgs def build_prompt(dur, times, subjects, schema): subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)" return PROMPT.format(nf=N_FRAMES, dur=dur, times=", ".join(f"{t:.2f}" for t in times), subjects=subs, schema=schema) def chat(processor, text): msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}] return processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) def run_transformers(model_id, prompt_text, imgs, max_new, four_bit): import torch from transformers import AutoModelForImageTextToText, AutoProcessor proc = AutoProcessor.from_pretrained(model_id) kw = {"dtype": torch.bfloat16, "device_map": "auto"} if four_bit: from transformers import BitsAndBytesConfig kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16) model = AutoModelForImageTextToText.from_pretrained(model_id, **kw).eval() enc = proc(text=[chat(proc, prompt_text)], images=imgs, return_tensors="pt").to(model.device) with torch.no_grad(): out = model.generate(**enc, max_new_tokens=max_new, do_sample=False) return proc.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True) def run_vllm(model_id, prompt_text, imgs, max_new): from transformers import AutoProcessor from vllm import LLM, SamplingParams proc = AutoProcessor.from_pretrained(model_id) llm = LLM(model=model_id, max_model_len=12288, limit_mm_per_prompt={"image": N_FRAMES}) o = llm.generate([{"prompt": chat(proc, prompt_text), "multi_modal_data": {"image": imgs}}], SamplingParams(temperature=0, max_tokens=max_new), use_tqdm=False)[0] return o.outputs[0].text def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("video") ap.add_argument("--subject", action="append", default=[], help='"ID: class (optional description)", repeatable') ap.add_argument("--model", default=REPO) ap.add_argument("--backend", choices=["transformers", "vllm"], default="transformers") ap.add_argument("--load-in-4bit", action="store_true", help="transformers + bitsandbytes NF4, fits small GPUs") ap.add_argument("--max-new-tokens", type=int, default=8192) ap.add_argument("-o", "--out", help="write the sheet here (default: stdout)") a = ap.parse_args() dur = duration(a.video) times, imgs = sample_frames(a.video, dur) text = build_prompt(dur, times, a.subject, schema_doc(a.model)) raw = (run_vllm(a.model, text, imgs, a.max_new_tokens) if a.backend == "vllm" else run_transformers(a.model, text, imgs, a.max_new_tokens, a.load_in_4bit)) m = re.search(r"\{.*\}", raw, re.S) try: sheet = json.loads(m.group(0)) if m else None except json.JSONDecodeError: sheet = None if sheet is None: print(raw, file=sys.stderr); sys.exit("could not parse a JSON sheet from the model output") s = json.dumps(sheet, indent=2) if a.out: open(a.out, "w").write(s + "\n") else: print(s) if __name__ == "__main__": main()