Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic-flash with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic-flash with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic-flash") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic-flash") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic-flash", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic-flash with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic-flash" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic-flash
- SGLang
How to use Cloudflare/clairic-flash with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic-flash" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic-flash" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic-flash", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic-flash with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic-flash
Download clairic_runtime/decoder_graph.py from Cloudflare/clairic-flash: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/Cloudflare/clairic-flash/resolve/main/clairic_runtime/decoder_graph.py
- Command line
-
hf download hf://Cloudflare/clairic-flash/clairic_runtime/decoder_graph.py
-
curl -L -o decoder_graph.py https://huggingface.co/Cloudflare/clairic-flash/resolve/main/clairic_runtime/decoder_graph.py
14.4 kB
| """Generic startup GDN decoder graphs with owned native state prefixes.""" | |
| from __future__ import annotations | |
| from contextlib import contextmanager | |
| import os | |
| import threading | |
| import time | |
| from types import MethodType | |
| from typing import Any, Iterator | |
| import weakref | |
| import torch | |
| from .decoder_capture import DecoderGraph, TorchBackend, callable_key | |
| from .gdn_pool import NativeStatePool | |
| class _SelectedCache: | |
| """Expose one real native layer to the proved single-layer capture adapter.""" | |
| def __init__(self, cache: Any, index: int) -> None: | |
| self.cache, self.index = cache, index | |
| self.layers = [cache.layers[index]] | |
| def has_previous_state(self, index: int) -> bool: | |
| assert index == 0 | |
| return self.cache.has_previous_state(self.index) | |
| def update_recurrent_state(self, state: torch.Tensor, index: int) -> torch.Tensor: | |
| assert index == 0 | |
| return self.cache.update_recurrent_state(state, self.index) | |
| class DecoderGraphs: | |
| """One engine, CUDA stream and worker; native prefill and dense attention stay eager.""" | |
| def __init__(self, backbone: Any, max_batch: int, buckets: tuple[int, ...]) -> None: | |
| self.backbone = backbone | |
| self.thread = threading.get_ident() | |
| self.device = backbone.device | |
| self.stream = torch.cuda.current_stream(self.device) | |
| self.max_batch, self.buckets = max_batch, buckets | |
| self.pool = NativeStatePool(backbone, max_batch) | |
| self.graphs: dict[tuple[int, int], DecoderGraph] = {} | |
| self.wrappers: list[tuple[Any, Any, Any]] = [] | |
| self.active: tuple[Any, int] | None = None | |
| self.closed = False | |
| self.preparing = True | |
| self.stats = {"replay": 0, "eager": 0, "retired": 0} | |
| self.cleanup: list[dict[str, Any]] = [] | |
| self.preparation: list[dict[str, Any]] = [] | |
| self.observe_requests = False | |
| self.last_request: dict[str, Any] | None = None | |
| self.active_methods: dict[int, Any] = {} | |
| def _owner(self) -> bool: | |
| return (not self.closed and threading.get_ident() == self.thread | |
| and torch.cuda.current_stream(self.device).cuda_stream == self.stream.cuda_stream) | |
| def _runtime() -> tuple[Any, ...]: | |
| import deep_gemm as dg | |
| return (dg.get_num_sms(), dg.get_tc_util(), dg.get_pdl(), | |
| tuple(sorted((key, value) for key, value in os.environ.items() if key.startswith("DG_")))) | |
| def prepare(self) -> None: | |
| if not self._owner(): | |
| raise RuntimeError("Graph preparation must use the engine CUDA worker") | |
| backend = TorchBackend(torch, self._runtime) | |
| try: | |
| for index in self.pool.states: | |
| layer = self.pool.language.layers[index] | |
| eager = layer.forward | |
| weak_manager = weakref.ref(self) | |
| def forward(this: Any, *args: Any, _index: int = index, | |
| _eager: Any = eager, _manager: Any = weak_manager, **kwargs: Any) -> Any: | |
| manager = _manager() | |
| if manager is None: | |
| return _eager(*args, **kwargs) | |
| return manager._forward(_index, _eager, args, kwargs) | |
| installed = MethodType(forward, layer) | |
| self.wrappers.append((layer, eager, installed)) | |
| layer.forward = installed | |
| dtype = torch.bfloat16 if index == 0 else torch.float32 | |
| for bucket in self.buckets: | |
| began = time.perf_counter_ns() | |
| dummy = self.pool.new_cache(bucket, previous=True) | |
| selected = _SelectedCache(dummy, index) | |
| def operation(value: Any, _eager: Any = eager, _cache: Any = dummy) -> Any: | |
| return _eager(value, position_embeddings=(), past_key_values=_cache, attention_mask=None) | |
| zero = torch.zeros(bucket, 1, layer.input_layernorm.weight.numel(), | |
| dtype=dtype, device=self.device) | |
| self.graphs[index, bucket] = DecoderGraph(backend, layer, selected, zero, | |
| operation, exclusive_process=True) | |
| entry = self.graphs[index, bucket].entry | |
| self.preparation.append({"layer": index, "bucket": bucket, "dtype": str(dtype), | |
| "capture_ms": (time.perf_counter_ns() - began) / 1e6, | |
| "generic_zero_input": True, | |
| "capture_mutation_restored": entry["capture_mutation_restored"], | |
| "pool": list(entry["pool"]), | |
| "pool_reserved_bytes": entry["pool_at_capture"]["reserved_bytes"], | |
| "static_input_storage_bytes": entry["static_input_storage_bytes"]}) | |
| self.pool.zero() | |
| self.stream.synchronize() | |
| self.preparing = False | |
| except BaseException: | |
| self.close() | |
| raise | |
| def request(self, batch: int) -> Iterator[Any]: | |
| if (not self._owner() or self.preparing or self.active is not None | |
| or not 1 <= batch <= self.max_batch or torch.is_grad_enabled() | |
| or torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing()): | |
| yield None | |
| return | |
| cache = self.pool.new_cache(batch) | |
| if not self.pool.owns(cache, batch): | |
| raise RuntimeError("Fresh cache must use owned native state prefixes") | |
| self.active = (cache, batch) | |
| self.active_methods = {index: self._cache_methods(cache, index) for index in self.pool.states} | |
| completed = False | |
| try: | |
| yield cache | |
| completed = True | |
| finally: | |
| try: | |
| if self.observe_requests: | |
| self.last_request = {"batch": batch, "completed": completed, | |
| "cache_type": type(cache).__name__, | |
| "native_prefixes_owned": self.pool.owns(cache, batch), | |
| "layers": [{"layer": index, "conv_shape": list(cache.layers[index].conv_states.shape), | |
| "conv_stride": list(cache.layers[index].conv_states.stride()), | |
| "conv_dtype": str(cache.layers[index].conv_states.dtype), | |
| "recurrent_shape": list(cache.layers[index].recurrent_states.shape), | |
| "recurrent_stride": list(cache.layers[index].recurrent_states.stride()), | |
| "recurrent_dtype": str(cache.layers[index].recurrent_states.dtype), | |
| "has_previous_state": bool(cache.layers[index].has_previous_state)} | |
| for index in self.pool.states]} | |
| finally: | |
| self.active = None | |
| self.active_methods.clear() | |
| def _cache_methods(cache: Any, index: int) -> tuple[Any, ...]: | |
| layer = cache.layers[index] | |
| return (callable_key(cache.update_recurrent_state), callable_key(cache.has_previous_state), | |
| callable_key(layer.update_recurrent_state), callable_key(layer.update_conv_state), | |
| layer.dtype, layer.device, layer.max_batch_size, layer.conv_kernel_size) | |
| def diagnostics(self) -> dict[str, Any]: | |
| """Primitive-only snapshot for separate untimed diagnostics and shutdown.""" | |
| return {"closed": self.closed, "preparing": self.preparing, | |
| "active": self.active is not None, "owner_thread": self.thread, | |
| "stream": self.stream.cuda_stream, "max_batch": self.max_batch, | |
| "buckets": list(self.buckets), "graphs": len(self.graphs), | |
| "arena_storage_bytes": self.pool.storage_bytes, | |
| "stats": dict(self.stats), "preparation": list(self.preparation), | |
| "observe_requests": self.observe_requests, | |
| "last_request": self.last_request, "cleanup": list(self.cleanup)} | |
| def _retire(self, index: int) -> None: | |
| for key in tuple(self.graphs): | |
| if key[0] == index: | |
| graph = self.graphs[key] | |
| details = graph.close() | |
| del self.graphs[key] | |
| self.cleanup.append({"layer": key[0], "bucket": key[1], **details}) | |
| if not details["passed"]: | |
| raise RuntimeError("Decoder graph retirement failed strict cleanup") | |
| self.stats["retired"] += 1 | |
| def _owns_layer(self, cache: Any, batch: int, index: int) -> bool: | |
| layer = cache.layers[index] | |
| for actual, owned in zip((layer.conv_states, layer.recurrent_states), self.pool.states[index], strict=True): | |
| if (actual is None or actual.data_ptr() != owned.data_ptr() | |
| or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr() | |
| or actual.dtype != owned.dtype or actual.device != owned.device | |
| or actual.storage_offset() != owned.storage_offset() | |
| or tuple(actual.shape) != (batch, *owned.shape[1:]) | |
| or tuple(actual.stride()) != tuple(owned.stride())): | |
| return False | |
| return layer.is_conv_states_initialized and layer.is_recurrent_states_initialized | |
| def _forward(self, index: int, eager: Any, args: Any, kwargs: Any) -> Any: | |
| value = args[0] if args else kwargs.get("hidden_states") | |
| cache = kwargs.get("past_key_values", args[4] if len(args) > 4 else None) | |
| mask = kwargs.get("attention_mask", args[2] if len(args) > 2 else None) | |
| batch = value.shape[0] if isinstance(value, torch.Tensor) and value.ndim == 3 else 0 | |
| expected_dtype = torch.bfloat16 if index == 0 else torch.float32 | |
| admitted = (not self.preparing and self._owner() and self.active is not None | |
| and cache is self.active[0] and batch == self.active[1] | |
| and value.shape[1] == 1 and value.is_contiguous() and value.dtype == expected_dtype | |
| and value.device == self.device and mask is None and not torch.is_grad_enabled() | |
| and not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing() | |
| and set(kwargs) <= {"hidden_states", "position_embeddings", "attention_mask", "position_ids", | |
| "past_key_values", "use_cache", "cache_position"} | |
| and self._cache_methods(cache, index) == self.active_methods.get(index) | |
| and self._owns_layer(cache, batch, index) and cache.has_previous_state(index)) | |
| if not admitted: | |
| self.stats["eager"] += 1 | |
| return eager(*args, **kwargs) | |
| bucket = next((bound for bound in self.buckets if batch <= bound), None) | |
| graph = self.graphs.get((index, bucket)) | |
| if graph is None: | |
| self.stats["eager"] += 1 | |
| return eager(*args, **kwargs) | |
| entry = graph.entry | |
| try: | |
| graph.check(entry["static_input"]) | |
| except RuntimeError: | |
| self._retire(index) | |
| self.stats["eager"] += 1 | |
| return eager(*args, **kwargs) | |
| entry["static_input"][:batch].copy_(value) | |
| if batch < bucket: | |
| entry["static_input"][batch:].zero_() | |
| entry["graph"].replay() | |
| self.stats["replay"] += 1 | |
| return entry["static_output"][:batch].clone() | |
| def close(self) -> None: | |
| if self.closed: | |
| return | |
| if not self._owner(): | |
| raise RuntimeError("Graph cleanup must use the engine CUDA worker") | |
| self.active = None | |
| self.active_methods.clear() | |
| error = None | |
| try: | |
| for index in tuple(self.pool.states): | |
| try: | |
| self._retire(index) | |
| except BaseException as exc: | |
| if error is None: | |
| error = exc | |
| finally: | |
| for layer, original, installed in reversed(self.wrappers): | |
| if layer.forward is installed: | |
| layer.forward = original | |
| self.wrappers.clear() | |
| self.pool.close() | |
| self.backbone = None | |
| self.closed = True | |
| if error is not None: | |
| raise error | |
| def _admissible(backbone: Any, max_batch: int) -> bool: | |
| """Reject unsupported contexts before allocating arenas or installing wrappers.""" | |
| config = backbone.config.get_text_config() | |
| limit = 64 if config.hidden_size == 4096 else 32 if config.hidden_size == 5120 else 0 | |
| if (backbone.device.type != "cuda" or backbone.dtype != torch.bfloat16 | |
| or not 1 <= max_batch <= limit): | |
| return False | |
| registry = torch.nn.modules.module | |
| if registry._global_forward_hooks or registry._global_forward_pre_hooks: | |
| return False | |
| for module in backbone.modules(): | |
| if module._forward_hooks or module._forward_pre_hooks: | |
| return False | |
| for layer in backbone.model.language_model.layers: | |
| if layer.layer_type != "linear_attention": | |
| continue | |
| for value in layer.parameters(): | |
| try: | |
| version = value._version | |
| except RuntimeError: | |
| return False | |
| if version is None: | |
| return False | |
| return not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing() | |
| def prepare(backbone: Any, max_batch: int) -> DecoderGraphs | None: | |
| """Run on the existing worker after math installation; unsupported engines stay eager.""" | |
| if not _admissible(backbone, max_batch): | |
| return None | |
| largest = next(bound for bound in (2, 8, 32, 64) if max_batch <= bound) | |
| buckets = tuple(bound for bound in (2, 8, 32, 64) if bound <= largest) | |
| with torch.no_grad(): | |
| manager = DecoderGraphs(backbone, max_batch=largest, buckets=buckets) | |
| manager.prepare() | |
| return manager | |