Image-Text-to-Text
Transformers
Safetensors
mage_vl
multimodal
vision-language-model
mage-vl
video-understanding
streaming
conversational
custom_code
Instructions to use microsoft/Mage-VL with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use microsoft/Mage-VL with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="microsoft/Mage-VL", trust_remote_code=True) messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# Load model directly from transformers import AutoModelForImageTextToText model = AutoModelForImageTextToText.from_pretrained("microsoft/Mage-VL", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use microsoft/Mage-VL with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "microsoft/Mage-VL" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "microsoft/Mage-VL", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/microsoft/Mage-VL
- SGLang
How to use microsoft/Mage-VL with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "microsoft/Mage-VL" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "microsoft/Mage-VL", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "microsoft/Mage-VL" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "microsoft/Mage-VL", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use microsoft/Mage-VL with Docker Model Runner:
docker model run hf.co/microsoft/Mage-VL
File size: 2,899 Bytes
12acbba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | """Single source of truth for DCVC canvas-selection parameters.
Reads the ``codec.dcvc`` block of ``preprocessor_config.json`` (found by walking
up from this file to the model directory). Used by ``dcvc_readiness_gen.py`` and
the readiness pipeline (``process_video_bitcost_readiness.py`` /
``process_video_bitcost_mv_mask_collage.py``) so ALL selection knobs are
controlled from the config file — NOT environment variables.
Full schema (defaults = the ``b50`` benchmark config) lives in
``preprocessor_config.json`` under ``codec.dcvc``.
"""
import json
import os
import functools
# Baseline defaults (the b50 benchmark config). Used only when a key is absent
# from preprocessor_config.json's codec.dcvc, so the file stays authoritative.
_DEFAULTS = {
# DCVC engine
"qp": 42, "reset_interval": 64, "intra_period": -1, "max_side": 0,
# readiness sampling / grouping
"num_sampled_frames": 256, "grouping_mode": "readiness",
"readiness_sum_threshold_mode": "auto", "group_size": 32,
"images_per_group": 4, "patch": 16, "max_pixels": 150000,
"min_group_frames": 8, "max_group_frames": 128,
"readiness_coverage_bins": 3, "readiness_delta_ratio": 0.05,
"bitcost_grid": "sub", "bitcost_pct": 99, "decode_backsearch_max": 16,
"canvas_format": "jpg",
# selection tuning knobs (our additions)
"per_frame_cap_ratio": 1.2, # spread block budget over more time frames
"bottom_atten": 0.5, # attenuate bottom-edge bit-cost (de-bias overlays)
"bottom_band": 0.10, # fraction of frame height treated as bottom band
"threshold_scale": 1.0, # scale readiness threshold (<1 -> more canvases)
"random_select": False, # random patch baseline (control)
"random_seed": 0,
}
@functools.lru_cache(maxsize=1)
def _load_file():
d = os.path.dirname(os.path.abspath(__file__))
for _ in range(4):
# preprocessor_config.json may sit next to this file (bundled layout) or in
# a sibling ``processor/`` subdir (multi-component model repo layout).
for p in (os.path.join(d, "preprocessor_config.json"),
os.path.join(d, "processor", "preprocessor_config.json")):
if os.path.exists(p):
try:
with open(p, encoding="utf-8") as f:
return json.load(f).get("codec", {}).get("dcvc", {}) or {}
except Exception:
return {}
d = os.path.dirname(d)
return {}
def get(key, cast=None):
"""Return codec.dcvc[key] from preprocessor_config.json, else the b50 default.
``cast`` optionally coerces (e.g. float, int, bool)."""
v = _load_file().get(key, _DEFAULTS.get(key))
if cast is not None and v is not None:
if cast is bool and isinstance(v, str):
return v.strip().lower() in ("1", "true", "yes")
return cast(v)
return v
|