shot-reader-8b / shot_reader.py
jhh3's picture
Shot Reader 8B: merged weights, LoRA adapter, inference script, model card
52e9143 verified
Raw History Blame Contribute Delete
6.01 kB
#!/usr/bin/env python3
"""Shot Reader: read one video shot and write its shot sheet (camera, subjects, paths, events, set) as JSON.
python shot_reader.py clip.mp4 --subject "P01: person (woman in a black polo)" --subject "O01: dog"
python shot_reader.py clip.mp4 --subject "P01: person" --backend vllm # fastest
python shot_reader.py clip.mp4 --subject "P01: person" --load-in-4bit # ~6 GB of VRAM
One shot = one continuous take (cut longer videos into shots first). The model sees 16 frames sampled
evenly across the shot at 448x256, plus the ids and classes of the subjects you name; it keeps your ids.
Needs: torch, transformers>=4.57, pillow, ffmpeg on PATH (+ vllm or bitsandbytes for those options).
"""
import argparse, json, os, re, subprocess, sys, tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
REPO = "voyager-so/shot-reader-8b"
N_FRAMES, SIZE = 16, (448, 256)
PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s
(frame i is at t = {times}).
Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres
in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward).
Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to
estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f}
(add intermediate keys when it moves). Keep the given ids.
Subjects in the brief (id: class): {subjects}
{schema}
Reply with only the JSON object, no prose, no code fences."""
def schema_doc(model):
p = os.path.join(HERE, "schema.md")
if not os.path.exists(p):
from huggingface_hub import hf_hub_download
p = hf_hub_download(model if "/" in model and not os.path.isdir(model) else REPO, "schema.md")
return open(p).read()
def duration(path):
out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path],
capture_output=True, text=True, check=True).stdout
return float(out.strip())
def sample_frames(path, dur):
from PIL import Image
times = [i * dur / N_FRAMES for i in range(N_FRAMES)]
imgs = []
with tempfile.TemporaryDirectory() as d:
for i, t in enumerate(times):
f = os.path.join(d, f"{i}.jpg")
subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path, "-frames:v", "1", "-q:v", "2", f],
check=True)
imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC))
return times, imgs
def build_prompt(dur, times, subjects, schema):
subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)"
return PROMPT.format(nf=N_FRAMES, dur=dur, times=", ".join(f"{t:.2f}" for t in times), subjects=subs, schema=schema)
def chat(processor, text):
msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}]
return processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False)
def run_transformers(model_id, prompt_text, imgs, max_new, four_bit):
import torch
from transformers import AutoModelForImageTextToText, AutoProcessor
proc = AutoProcessor.from_pretrained(model_id)
kw = {"dtype": torch.bfloat16, "device_map": "auto"}
if four_bit:
from transformers import BitsAndBytesConfig
kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.bfloat16)
model = AutoModelForImageTextToText.from_pretrained(model_id, **kw).eval()
enc = proc(text=[chat(proc, prompt_text)], images=imgs, return_tensors="pt").to(model.device)
with torch.no_grad():
out = model.generate(**enc, max_new_tokens=max_new, do_sample=False)
return proc.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True)
def run_vllm(model_id, prompt_text, imgs, max_new):
from transformers import AutoProcessor
from vllm import LLM, SamplingParams
proc = AutoProcessor.from_pretrained(model_id)
llm = LLM(model=model_id, max_model_len=12288, limit_mm_per_prompt={"image": N_FRAMES})
o = llm.generate([{"prompt": chat(proc, prompt_text), "multi_modal_data": {"image": imgs}}],
SamplingParams(temperature=0, max_tokens=max_new), use_tqdm=False)[0]
return o.outputs[0].text
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("video")
ap.add_argument("--subject", action="append", default=[], help='"ID: class (optional description)", repeatable')
ap.add_argument("--model", default=REPO)
ap.add_argument("--backend", choices=["transformers", "vllm"], default="transformers")
ap.add_argument("--load-in-4bit", action="store_true", help="transformers + bitsandbytes NF4, fits small GPUs")
ap.add_argument("--max-new-tokens", type=int, default=8192)
ap.add_argument("-o", "--out", help="write the sheet here (default: stdout)")
a = ap.parse_args()
dur = duration(a.video)
times, imgs = sample_frames(a.video, dur)
text = build_prompt(dur, times, a.subject, schema_doc(a.model))
raw = (run_vllm(a.model, text, imgs, a.max_new_tokens) if a.backend == "vllm"
else run_transformers(a.model, text, imgs, a.max_new_tokens, a.load_in_4bit))
m = re.search(r"\{.*\}", raw, re.S)
try:
sheet = json.loads(m.group(0)) if m else None
except json.JSONDecodeError:
sheet = None
if sheet is None:
print(raw, file=sys.stderr); sys.exit("could not parse a JSON sheet from the model output")
s = json.dumps(sheet, indent=2)
if a.out: open(a.out, "w").write(s + "\n")
else: print(s)
if __name__ == "__main__":
main()