Instructions to use nvidia/Flora-Captioner with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use nvidia/Flora-Captioner with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("image-text-to-text", model="nvidia/Flora-Captioner")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
pipe(text=messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoProcessor, AutoModelForMultimodalLM

processor = AutoProcessor.from_pretrained("nvidia/Flora-Captioner")
model = AutoModelForMultimodalLM.from_pretrained("nvidia/Flora-Captioner", device_map="auto")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
inputs = processor.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=256)
print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use nvidia/Flora-Captioner with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "nvidia/Flora-Captioner"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "nvidia/Flora-Captioner",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker

docker model run hf.co/nvidia/Flora-Captioner

SGLang

How to use nvidia/Flora-Captioner with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "nvidia/Flora-Captioner" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "nvidia/Flora-Captioner",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "nvidia/Flora-Captioner" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "nvidia/Flora-Captioner",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Docker Model Runner
How to use nvidia/Flora-Captioner with Docker Model Runner:
```
docker model run hf.co/nvidia/Flora-Captioner
```

Flora-Captioner / batch_utils /vendor /MANIFEST.json

Omar L

Add Flora inference examples (#1)

00c7586 6 days ago

Raw History Blame Contribute Delete

3.23 kB

{
  "flora_unified/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
  "flora_unified/unified_input.py": "2aa2b521191b68913b03d7582b7a3bcfad49d90c014a1a1bb7571b5e4f08d7ef",
  "production_adapter/TEMPSEG_PROMPTS.json": "66edd6fb43b7fa55cc6257beee5c42a1a4555ab9790320c31114d77bd07a05f1",
  "production_adapter/adapter.py": "083c6788b561d726a85d9f3f8f516593a9f6f76a43f315581c821871e3f82312",
  "production_adapter/core/flora_runtime/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
  "production_adapter/core/flora_runtime/exact_frame_serving.py": "f921e279b5d52f723b3e77e25441763683d05c6e9cd2f8ff204466a50255b346",
  "production_adapter/core/flora_runtime/schema.py": "42c523e4db1a4da3fe0d0d578f3cf6e94aed7666c06e3f18081c209159de04b9",
  "production_adapter/core/flora_runtime/subtask_captioning/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
  "production_adapter/core/flora_runtime/subtask_captioning/u2_contract.py": "873476e78a9e3e222ae9e47fcb7a04c8c18ef518b2847c401a2e1cfb2292fe90",
  "production_adapter/core/flora_runtime/subtask_captioning/u2_worker.py": "870641228b0be442613d520232d451047742ca2ea41fd0cd51dbf0a58ba567d2",
  "production_adapter/core/flora_runtime/video_segmentation/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
  "production_adapter/core/flora_runtime/video_segmentation/contracts.py": "8591d849b0b65fda6b21511557d4ee5eabd47f8332e2b6831f40d5bb472d978c",
  "production_adapter/core/flora_runtime/video_segmentation/fine_subtask.py": "6fbf8337bab8491148a78a822187d74397b7f0552700252451a18cd7c78c8e04",
  "production_adapter/core/flora_runtime/video_segmentation/fine_temporal.py": "32f2b99c97176f5e10934450da638577268c2e5d9977c2ea138ebf1b7eda58fa",
  "production_adapter/core/flora_runtime/video_segmentation/fine_worker.py": "8a31a77180c8d406113d4b494644153564a6f46c8a9b68ad7babb3cc1695edec",
  "production_adapter/core/flora_runtime/video_segmentation/interface.py": "a5424b2cc3a4c879552dd49bae207a58fb9e751359f441c03af167a2e1879359",
  "production_adapter/core/flora_runtime/video_segmentation/scope_scheduler.py": "be95411b39124d83811fe74dcac2d911169ecf475a81b7460c64b19fbbc7e5db",
  "production_adapter/core/flora_runtime/worker_transport.py": "30fb64869eb6a27ab96bf394c105f19fcb42590be4424c66b76b89e17631693a",
  "production_gdn_fix.py": "a3f52604bfa2ec58901df63215df5e47d37e1d7974aa56d36a1e4efe8cad3fc6",
  "pts_input_adapter.py": "d784d6b77005eb6945dee334e4c9136fbededfa2049d5945244f198517984767",
  "robot_caption_geometry.py": "1c874d3c63359663276d31a3e9c3112a5982ad4000f44c808541b2471f5f7d2d",
  "support/canonical_rgb.py": "1e3230c60b45d8e017060d8a5dd2b5f48e2199192f09dcff4d04c05b4349e35a",
  "support/composers.py": "180b97a6a634979c1b0dbe720ed7a1abd818dc7ee31fdc4eeca7a8277e43d1b6",
  "support/plans.py": "d75b918804c9ec10f8642d362a54a74c3eb875abf22e8a60228a0d6b2cbb570e",
  "support/u2_contract.py": "50190914b719bbc9b8f2b53e52a6bd2b142a452e2fb1c5c897ea4c31123ad4dd",
  "tempseg_fp32_loader.py": "7ee9889f5aa5909abf3ecd1c34e3ac2f81a32a2098d94ae3ba9f33831e7a07c0",
  "verify_gdn.py": "96596ff2db4425f4dea71798aeabb581aa83e1ad8a878cfcb9efb8dbac80cda6"
}

	{
	"flora_unified/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
	"flora_unified/unified_input.py": "2aa2b521191b68913b03d7582b7a3bcfad49d90c014a1a1bb7571b5e4f08d7ef",
	"production_adapter/TEMPSEG_PROMPTS.json": "66edd6fb43b7fa55cc6257beee5c42a1a4555ab9790320c31114d77bd07a05f1",
	"production_adapter/adapter.py": "083c6788b561d726a85d9f3f8f516593a9f6f76a43f315581c821871e3f82312",
	"production_adapter/core/flora_runtime/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
	"production_adapter/core/flora_runtime/exact_frame_serving.py": "f921e279b5d52f723b3e77e25441763683d05c6e9cd2f8ff204466a50255b346",
	"production_adapter/core/flora_runtime/schema.py": "42c523e4db1a4da3fe0d0d578f3cf6e94aed7666c06e3f18081c209159de04b9",
	"production_adapter/core/flora_runtime/subtask_captioning/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
	"production_adapter/core/flora_runtime/subtask_captioning/u2_contract.py": "873476e78a9e3e222ae9e47fcb7a04c8c18ef518b2847c401a2e1cfb2292fe90",
	"production_adapter/core/flora_runtime/subtask_captioning/u2_worker.py": "870641228b0be442613d520232d451047742ca2ea41fd0cd51dbf0a58ba567d2",
	"production_adapter/core/flora_runtime/video_segmentation/__init__.py": "9f56e761d79bfdb34304a012586cb04d16b435ef6130091a97702e559260a2f2",
	"production_adapter/core/flora_runtime/video_segmentation/contracts.py": "8591d849b0b65fda6b21511557d4ee5eabd47f8332e2b6831f40d5bb472d978c",
	"production_adapter/core/flora_runtime/video_segmentation/fine_subtask.py": "6fbf8337bab8491148a78a822187d74397b7f0552700252451a18cd7c78c8e04",
	"production_adapter/core/flora_runtime/video_segmentation/fine_temporal.py": "32f2b99c97176f5e10934450da638577268c2e5d9977c2ea138ebf1b7eda58fa",
	"production_adapter/core/flora_runtime/video_segmentation/fine_worker.py": "8a31a77180c8d406113d4b494644153564a6f46c8a9b68ad7babb3cc1695edec",
	"production_adapter/core/flora_runtime/video_segmentation/interface.py": "a5424b2cc3a4c879552dd49bae207a58fb9e751359f441c03af167a2e1879359",
	"production_adapter/core/flora_runtime/video_segmentation/scope_scheduler.py": "be95411b39124d83811fe74dcac2d911169ecf475a81b7460c64b19fbbc7e5db",
	"production_adapter/core/flora_runtime/worker_transport.py": "30fb64869eb6a27ab96bf394c105f19fcb42590be4424c66b76b89e17631693a",
	"production_gdn_fix.py": "a3f52604bfa2ec58901df63215df5e47d37e1d7974aa56d36a1e4efe8cad3fc6",
	"pts_input_adapter.py": "d784d6b77005eb6945dee334e4c9136fbededfa2049d5945244f198517984767",
	"robot_caption_geometry.py": "1c874d3c63359663276d31a3e9c3112a5982ad4000f44c808541b2471f5f7d2d",
	"support/canonical_rgb.py": "1e3230c60b45d8e017060d8a5dd2b5f48e2199192f09dcff4d04c05b4349e35a",
	"support/composers.py": "180b97a6a634979c1b0dbe720ed7a1abd818dc7ee31fdc4eeca7a8277e43d1b6",
	"support/plans.py": "d75b918804c9ec10f8642d362a54a74c3eb875abf22e8a60228a0d6b2cbb570e",
	"support/u2_contract.py": "50190914b719bbc9b8f2b53e52a6bd2b142a452e2fb1c5c897ea4c31123ad4dd",
	"tempseg_fp32_loader.py": "7ee9889f5aa5909abf3ecd1c34e3ac2f81a32a2098d94ae3ba9f33831e7a07c0",
	"verify_gdn.py": "96596ff2db4425f4dea71798aeabb581aa83e1ad8a878cfcb9efb8dbac80cda6"
	}