Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic
- SGLang
How to use Cloudflare/clairic with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic
Download clairic_runtime/attention_metadata.py from Cloudflare/clairic: direct link, hf CLI and curl.
- Browser
- Download file 7.61 kB
-
https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/attention_metadata.py
- Command line
-
hf download hf://Cloudflare/clairic/clairic_runtime/attention_metadata.py
-
curl -L -o attention_metadata.py https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/attention_metadata.py
7.61 kB
| """Reuse pinned FA3 unpadding metadata within one owned decode forward.""" | |
| from __future__ import annotations | |
| from collections.abc import Callable, Iterator | |
| from contextlib import contextmanager | |
| from contextvars import ContextVar | |
| import threading | |
| from typing import Any | |
| import torch | |
| from transformers.integrations.flash_attention import flash_attention_forward | |
| from transformers.modeling_flash_attention_utils import _get_unpad_data, _index_first_axis | |
| _PINNED_FA3 = "kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0" | |
| _VARLEN_KEYS = frozenset({"cu_seq_lens_q", "cu_seq_lens_k", "max_length_q", "max_length_k"}) | |
| class _Entry: | |
| def __init__(self) -> None: | |
| self.key: tuple[Any, ...] | None = None | |
| self.mask: torch.Tensor | None = None | |
| self.indices: torch.Tensor | None = None | |
| self.cu_k: torch.Tensor | None = None | |
| self.cu_q: torch.Tensor | None = None | |
| self.maximum = 0 | |
| self.bytes = 0 | |
| def clear(self) -> None: | |
| self.key, self.mask, self.indices, self.cu_k, self.cu_q = None, None, None, None, None | |
| self.maximum, self.bytes = 0, 0 | |
| class DecodeMetadata: | |
| def __init__(self, max_bytes: int = 8 * 1024**2) -> None: | |
| if max_bytes <= 0: | |
| raise ValueError("Expected a positive metadata bound") | |
| self.max_bytes = max_bytes | |
| self._lock = threading.Lock() | |
| self._current: ContextVar[_Entry | None] = ContextVar("clairic_decode_metadata", default=None) | |
| self._active: _Entry | None = None | |
| self._closed = False | |
| self.scopes = 0 | |
| self.materializations = 0 | |
| self.hits = 0 | |
| self.peak_bytes = 0 | |
| def scope(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Iterator[None]: | |
| token = self._current.set(None) | |
| acquired = False | |
| entry = None | |
| try: | |
| inputs = kwargs.get("input_ids", args[0] if args else None) | |
| if inputs is None: | |
| inputs = kwargs.get("inputs_embeds") | |
| eligible = (not self._closed and not torch.is_grad_enabled() | |
| and torch.is_inference_mode_enabled() and not torch.compiler.is_compiling() | |
| and isinstance(inputs, torch.Tensor) and inputs.is_cuda | |
| and inputs.ndim in (2, 3) and inputs.shape[1] == 1) | |
| if eligible and self._lock.acquire(blocking=False): | |
| acquired = True | |
| if not self._closed: | |
| entry = _Entry() | |
| self._active = entry | |
| self._current.set(entry) | |
| self.scopes += 1 | |
| yield | |
| finally: | |
| if entry is not None: | |
| entry.clear() | |
| self._active = None | |
| self._current.reset(token) | |
| if acquired: | |
| self._lock.release() | |
| def _metadata(self, mask: torch.Tensor) -> _Entry | None: | |
| entry = self._current.get() | |
| if self._closed or entry is None: | |
| return None | |
| batch = mask.shape[0] | |
| if mask.numel() * 8 + 2 * (batch + 1) * 4 > self.max_bytes: | |
| entry.clear() | |
| return None | |
| try: | |
| version = mask._version | |
| except RuntimeError: | |
| version = None | |
| key = (id(mask), version, mask.data_ptr(), mask.storage_offset(), mask.device, | |
| mask.dtype, tuple(mask.shape), tuple(mask.stride()), | |
| torch.cuda.current_stream(mask.device).cuda_stream) | |
| if entry.mask is mask and entry.key == key: | |
| self.hits += 1 | |
| return entry | |
| entry.clear() | |
| indices, cu_k, maximum = _get_unpad_data(mask) | |
| cu_q = torch.arange(batch + 1, dtype=torch.int32, device=mask.device) | |
| retained = sum(value.numel() * value.element_size() for value in (indices, cu_k, cu_q)) | |
| if retained > self.max_bytes: | |
| return None | |
| entry.key, entry.mask = key, mask | |
| entry.indices, entry.cu_k, entry.cu_q = indices, cu_k, cu_q | |
| entry.maximum, entry.bytes = int(maximum.item()), retained | |
| self.materializations += 1 | |
| self.peak_bytes = max(self.peak_bytes, retained) | |
| return entry | |
| def attend(self, interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]], | |
| module: Any, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, | |
| mask: torch.Tensor | None, *, dropout: float, scaling: float, | |
| **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]: | |
| eligible = (not self._closed and self._current.get() is not None | |
| and not torch.is_grad_enabled() and torch.is_inference_mode_enabled() | |
| and not torch.compiler.is_compiling() and interface is flash_attention_forward | |
| and module.config._attn_implementation == _PINNED_FA3 | |
| and dropout == 0.0 | |
| and getattr(module.config, "sliding_window", None) is None | |
| and kwargs.get("sliding_window") is None and not (_VARLEN_KEYS & kwargs.keys()) | |
| and mask is not None and mask.is_cuda and mask.ndim == 2 | |
| and mask.dtype == torch.bool and mask.numel() > 0 | |
| and all(item.is_cuda and item.dtype == torch.bfloat16 and item.ndim == 4 | |
| and item.device == mask.device and item.stride(-1) == 1 | |
| for item in (query, key, value))) | |
| if eligible: | |
| assert mask is not None | |
| positions = kwargs.get("position_ids") | |
| eligible = (query.shape[0] == key.shape[0] == value.shape[0] == mask.shape[0] | |
| and query.shape[2] == 1 and key.shape[2] == value.shape[2] == mask.shape[1] | |
| and mask.shape[1] > 1 | |
| and query.shape[1] in (16, 24) and key.shape[1] == value.shape[1] == 4 | |
| and query.shape[3] == key.shape[3] == value.shape[3] == 256 | |
| and (positions is None or (isinstance(positions, torch.Tensor) | |
| and positions.ndim == 2 and tuple(positions.shape) == (query.shape[0], 1)))) | |
| entry = self._metadata(mask) if eligible and mask is not None else None | |
| if entry is None: | |
| active = self._current.get() | |
| if active is not None: | |
| active.clear() | |
| return interface(module, query, key, value, mask, dropout=dropout, scaling=scaling, **kwargs) | |
| assert entry.indices is not None and entry.cu_q is not None and entry.cu_k is not None | |
| packed_key = _index_first_axis(key.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2) | |
| packed_value = _index_first_axis(value.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2) | |
| return interface(module, query, packed_key, packed_value, None, dropout=dropout, scaling=scaling, | |
| cu_seq_lens_q=entry.cu_q, cu_seq_lens_k=entry.cu_k, | |
| max_length_q=1, max_length_k=entry.maximum, **kwargs) | |
| def snapshot(self) -> dict[str, int | bool]: | |
| return {"scopes": self.scopes, "materializations": self.materializations, "hits": self.hits, | |
| "peak_bytes": self.peak_bytes, "retained_bytes": self._active.bytes if self._active else 0, | |
| "closed": self._closed} | |
| def close(self) -> None: | |
| self._closed = True | |
| with self._lock: | |
| if self._active is not None: | |
| self._active.clear() | |
| self._active = None | |