Image-Text-to-Text
Transformers
Safetensors
English
qwen3_vl
vision
video-understanding
previs
previsualization
camera
3d
blender
scene-understanding
qwen3-vl
lora
conversational
Instructions to use voyager-so/shot-reader-8b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use voyager-so/shot-reader-8b with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="voyager-so/shot-reader-8b") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("voyager-so/shot-reader-8b") model = AutoModelForMultimodalLM.from_pretrained("voyager-so/shot-reader-8b", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use voyager-so/shot-reader-8b with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "voyager-so/shot-reader-8b" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "voyager-so/shot-reader-8b", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/voyager-so/shot-reader-8b
- SGLang
How to use voyager-so/shot-reader-8b with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "voyager-so/shot-reader-8b" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "voyager-so/shot-reader-8b", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "voyager-so/shot-reader-8b" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "voyager-so/shot-reader-8b", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use voyager-so/shot-reader-8b with Docker Model Runner:
docker model run hf.co/voyager-so/shot-reader-8b
Download shot_reader.py from voyager-so/shot-reader-8b: direct link, hf CLI and curl.
- Browser
- Download file 6.01 kB
-
https://huggingface.co/voyager-so/shot-reader-8b/resolve/main/shot_reader.py
- Command line
-
hf download hf://voyager-so/shot-reader-8b/shot_reader.py
-
curl -L -o shot_reader.py https://huggingface.co/voyager-so/shot-reader-8b/resolve/main/shot_reader.py
6.01 kB
| #!/usr/bin/env python3 | |
| """Shot Reader: read one video shot and write its shot sheet (camera, subjects, paths, events, set) as JSON. | |
| python shot_reader.py clip.mp4 --subject "P01: person (woman in a black polo)" --subject "O01: dog" | |
| python shot_reader.py clip.mp4 --subject "P01: person" --backend vllm # fastest | |
| python shot_reader.py clip.mp4 --subject "P01: person" --load-in-4bit # ~6 GB of VRAM | |
| One shot = one continuous take (cut longer videos into shots first). The model sees 16 frames sampled | |
| evenly across the shot at 448x256, plus the ids and classes of the subjects you name; it keeps your ids. | |
| Needs: torch, transformers>=4.57, pillow, ffmpeg on PATH (+ vllm or bitsandbytes for those options). | |
| """ | |
| import argparse, json, os, re, subprocess, sys, tempfile | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| REPO = "voyager-so/shot-reader-8b" | |
| N_FRAMES, SIZE = 16, (448, 256) | |
| PROMPT = """You are a previz shot reader. You see {nf} frames sampled evenly from one shot of {dur:.2f} s | |
| (frame i is at t = {times}). | |
| Write the shot's fact sheet: one JSON object exactly in the schema below. Measure positions in metres | |
| in the ground frame defined below (origin under the camera in the FIRST frame, +x right, +y forward). | |
| Use object sizes (people are about 1.7 m tall, doors 2 m, chairs 0.9 m, etc.) and perspective to | |
| estimate distances. Give every listed subject a path with at least a key at t=0 and t={dur:.2f} | |
| (add intermediate keys when it moves). Keep the given ids. | |
| Subjects in the brief (id: class): {subjects} | |
| {schema} | |
| Reply with only the JSON object, no prose, no code fences.""" | |
| def schema_doc(model): | |
| p = os.path.join(HERE, "schema.md") | |
| if not os.path.exists(p): | |
| from huggingface_hub import hf_hub_download | |
| p = hf_hub_download(model if "/" in model and not os.path.isdir(model) else REPO, "schema.md") | |
| return open(p).read() | |
| def duration(path): | |
| out = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", path], | |
| capture_output=True, text=True, check=True).stdout | |
| return float(out.strip()) | |
| def sample_frames(path, dur): | |
| from PIL import Image | |
| times = [i * dur / N_FRAMES for i in range(N_FRAMES)] | |
| imgs = [] | |
| with tempfile.TemporaryDirectory() as d: | |
| for i, t in enumerate(times): | |
| f = os.path.join(d, f"{i}.jpg") | |
| subprocess.run(["ffmpeg", "-v", "error", "-y", "-ss", f"{t:.3f}", "-i", path, "-frames:v", "1", "-q:v", "2", f], | |
| check=True) | |
| imgs.append(Image.open(f).convert("RGB").resize(SIZE, Image.BICUBIC)) | |
| return times, imgs | |
| def build_prompt(dur, times, subjects, schema): | |
| subs = ", ".join(subjects) if subjects else "none given (name people P01, P02, ... and objects by class)" | |
| return PROMPT.format(nf=N_FRAMES, dur=dur, times=", ".join(f"{t:.2f}" for t in times), subjects=subs, schema=schema) | |
| def chat(processor, text): | |
| msgs = [{"role": "user", "content": [{"type": "image"}] * N_FRAMES + [{"type": "text", "text": text}]}] | |
| return processor.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False) | |
| def run_transformers(model_id, prompt_text, imgs, max_new, four_bit): | |
| import torch | |
| from transformers import AutoModelForImageTextToText, AutoProcessor | |
| proc = AutoProcessor.from_pretrained(model_id) | |
| kw = {"dtype": torch.bfloat16, "device_map": "auto"} | |
| if four_bit: | |
| from transformers import BitsAndBytesConfig | |
| kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", | |
| bnb_4bit_compute_dtype=torch.bfloat16) | |
| model = AutoModelForImageTextToText.from_pretrained(model_id, **kw).eval() | |
| enc = proc(text=[chat(proc, prompt_text)], images=imgs, return_tensors="pt").to(model.device) | |
| with torch.no_grad(): | |
| out = model.generate(**enc, max_new_tokens=max_new, do_sample=False) | |
| return proc.tokenizer.decode(out[0, enc["input_ids"].shape[1]:], skip_special_tokens=True) | |
| def run_vllm(model_id, prompt_text, imgs, max_new): | |
| from transformers import AutoProcessor | |
| from vllm import LLM, SamplingParams | |
| proc = AutoProcessor.from_pretrained(model_id) | |
| llm = LLM(model=model_id, max_model_len=12288, limit_mm_per_prompt={"image": N_FRAMES}) | |
| o = llm.generate([{"prompt": chat(proc, prompt_text), "multi_modal_data": {"image": imgs}}], | |
| SamplingParams(temperature=0, max_tokens=max_new), use_tqdm=False)[0] | |
| return o.outputs[0].text | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("video") | |
| ap.add_argument("--subject", action="append", default=[], help='"ID: class (optional description)", repeatable') | |
| ap.add_argument("--model", default=REPO) | |
| ap.add_argument("--backend", choices=["transformers", "vllm"], default="transformers") | |
| ap.add_argument("--load-in-4bit", action="store_true", help="transformers + bitsandbytes NF4, fits small GPUs") | |
| ap.add_argument("--max-new-tokens", type=int, default=8192) | |
| ap.add_argument("-o", "--out", help="write the sheet here (default: stdout)") | |
| a = ap.parse_args() | |
| dur = duration(a.video) | |
| times, imgs = sample_frames(a.video, dur) | |
| text = build_prompt(dur, times, a.subject, schema_doc(a.model)) | |
| raw = (run_vllm(a.model, text, imgs, a.max_new_tokens) if a.backend == "vllm" | |
| else run_transformers(a.model, text, imgs, a.max_new_tokens, a.load_in_4bit)) | |
| m = re.search(r"\{.*\}", raw, re.S) | |
| try: | |
| sheet = json.loads(m.group(0)) if m else None | |
| except json.JSONDecodeError: | |
| sheet = None | |
| if sheet is None: | |
| print(raw, file=sys.stderr); sys.exit("could not parse a JSON sheet from the model output") | |
| s = json.dumps(sheet, indent=2) | |
| if a.out: open(a.out, "w").write(s + "\n") | |
| else: print(s) | |
| if __name__ == "__main__": | |
| main() | |