Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic
- SGLang
How to use Cloudflare/clairic with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic
Download clairic_runtime/decoder_capture.py from Cloudflare/clairic: direct link, hf CLI and curl.
- Browser
- Download file 11.1 kB
-
https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/decoder_capture.py
- Command line
-
hf download hf://Cloudflare/clairic/clairic_runtime/decoder_capture.py
-
curl -L -o decoder_capture.py https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/decoder_capture.py
11.1 kB
| """Owned single-layer graph capture and strict resource lifecycle.""" | |
| from __future__ import annotations | |
| import threading | |
| from typing import Any | |
| import weakref | |
| def tensor_key(value: Any, *, version: bool) -> tuple[Any, ...]: | |
| try: | |
| counter = value._version if version else None | |
| except RuntimeError: | |
| counter = None | |
| storage = value.untyped_storage() | |
| return (id(value), counter, value.data_ptr(), storage.data_ptr(), storage.nbytes(), | |
| value.storage_offset(), str(value.device), str(value.dtype), | |
| tuple(value.shape), tuple(value.stride())) | |
| def callable_key(value: Any) -> tuple[int, int | None]: | |
| return (id(getattr(value, "__func__", value)), | |
| id(value.__self__) if hasattr(value, "__self__") else None) | |
| def input_key(value: Any) -> tuple[Any, ...]: | |
| return str(value.device), str(value.dtype), tuple(value.shape), tuple(value.stride()) | |
| def owner_key(torch: Any, owner: Any, cache: Any, operation: Any) -> tuple[Any, ...]: | |
| modules = [] | |
| for name, module in owner.named_modules(): | |
| if module._forward_hooks or module._forward_pre_hooks: | |
| raise RuntimeError("Forward hooks invalidate decoder capture") | |
| scalars = tuple(sorted((key, value) for key, value in vars(module).items() | |
| if not key.startswith("_") and isinstance(value, (str, bool, int, float)))) | |
| callables = tuple(sorted((key, callable_key(value)) for key, value in vars(module).items() | |
| if not key.startswith("_") and callable(value))) | |
| modules.append((name, id(module), callable_key(module.forward), scalars, callables)) | |
| registry = torch.nn.modules.module | |
| if registry._global_forward_hooks or registry._global_forward_pre_hooks: | |
| raise RuntimeError("Global forward hooks invalidate decoder capture") | |
| parameters = tuple((name, tensor_key(value, version=True)) | |
| for name, value in owner.named_parameters()) | |
| if any(item[1][1] is None for item in parameters): | |
| raise RuntimeError("Versioned parameters required") | |
| buffers = tuple((name, tensor_key(value, version=True)) | |
| for name, value in owner.named_buffers()) | |
| layer = cache.layers[0] | |
| if not (cache.has_previous_state(0) and layer.is_conv_states_initialized | |
| and layer.is_recurrent_states_initialized): | |
| raise RuntimeError("Already initialized live decode cache required") | |
| states = tuple(tensor_key(value, version=False) | |
| for value in (layer.conv_states, layer.recurrent_states)) | |
| return (id(owner), id(cache), id(layer), callable_key(operation), tuple(modules), | |
| parameters, buffers, states, callable_key(cache.update_recurrent_state), | |
| callable_key(cache.has_previous_state), callable_key(layer.update_recurrent_state)) | |
| def describe_pool(segments: list[dict[str, Any]], pool: tuple[int, int]) -> dict[str, Any]: | |
| blocks = [block for segment in segments for block in segment["blocks"]] | |
| pending = {"active_awaiting_free", "active_pending_free"} | |
| return {"pool": pool, "segments": segments, | |
| "foreign_pool": any(tuple(segment["segment_pool_id"]) != pool for segment in segments), | |
| "reserved_bytes": sum(segment["total_size"] for segment in segments), | |
| "allocated_bytes": sum(segment["allocated_size"] for segment in segments), | |
| "active_bytes": sum(segment["active_size"] for segment in segments), | |
| "active_allocated_blocks": sum(block["state"] == "active_allocated" for block in blocks), | |
| "pending_blocks": sum(block["state"] in pending for block in blocks), | |
| "unknown_block_states": sorted({block["state"] for block in blocks} | |
| - pending - {"active_allocated", "inactive"})} | |
| class DecoderGraph: | |
| """One exact bucket/dtype graph with owned input and cloned FP32 outputs. | |
| Capture/replay must run on the engine's exclusive CUDA worker. Mutation is | |
| intentional only for the two captured cache tensors. Any owner, parameter, | |
| callable, stream, runtime, cache storage or hook change rejects the session. | |
| Capture/replay failures propagate; callers must close the engine rather than | |
| retrying eager work on a possibly invalid CUDA context. | |
| """ | |
| def __init__(self, backend: Any, owner: Any, cache: Any, value: Any, | |
| operation: Any, *, exclusive_process: bool) -> None: | |
| if not exclusive_process: | |
| raise ValueError("Exclusive process required") | |
| self.backend, self.owner, self.cache, self.operation = backend, owner, cache, operation | |
| self.thread, self.closed = threading.get_ident(), False | |
| self.context = backend.context(value) | |
| self.input = input_key(value) | |
| self.key = owner_key(backend.torch, owner, cache, operation) | |
| self.entry = backend.capture(value, operation, cache) | |
| self.check(value) | |
| def check(self, value: Any) -> None: | |
| if self.closed or self.thread != threading.get_ident(): | |
| raise RuntimeError("Closed graph or foreign thread") | |
| if self.backend.context(value) != self.context or input_key(value) != self.input: | |
| raise RuntimeError("Input layout, dtype, stream or runtime changed") | |
| if owner_key(self.backend.torch, self.owner, self.cache, self.operation) != self.key: | |
| raise RuntimeError("Decoder owner, parameter, callable or cache changed") | |
| def run(self, value: Any) -> Any: | |
| self.check(value) | |
| return self.direct(value) | |
| def direct(self, value: Any) -> Any: | |
| # The padded manager executes complete guards before direct replay. | |
| return self.backend.replay_clone(self.entry, value) | |
| def close(self) -> dict[str, Any]: | |
| if self.closed or self.thread != threading.get_ident(): | |
| raise RuntimeError("Closed graph or foreign thread") | |
| details = self.backend.release(self.entry) | |
| self.owner = self.cache = self.operation = self.key = None | |
| self.closed = True | |
| return details | |
| class TorchBackend: | |
| def __init__(self, torch: Any, runtime_key: Any) -> None: | |
| self.torch, self.runtime_key = torch, runtime_key | |
| def context(self, value: Any) -> tuple[Any, ...]: | |
| torch = self.torch | |
| if (torch.is_grad_enabled() or torch.compiler.is_compiling() or not value.is_cuda | |
| or torch.cuda.is_current_stream_capturing() or not value.is_contiguous() | |
| or value.dtype not in (torch.bfloat16, torch.float32) | |
| or value.ndim != 3 or value.shape[1] != 1 or min(value.shape) < 1): | |
| raise RuntimeError("Unsupported decoder graph context/input") | |
| return (torch.cuda.current_stream(value.device).cuda_stream, self.runtime_key()) | |
| def capture(self, value: Any, operation: Any, cache: Any) -> dict[str, Any]: | |
| torch = self.torch | |
| current = torch.cuda.current_stream(value.device) | |
| side = torch.cuda.Stream(device=value.device) | |
| side.wait_stream(current) | |
| layer = cache.layers[0] | |
| states = (layer.conv_states, layer.recurrent_states) | |
| saved = tuple(state.clone(memory_format=torch.preserve_format) for state in states) | |
| side.wait_stream(current) | |
| def restore(current_states: Any, before_states: Any) -> None: | |
| for state, before in zip(current_states, before_states, strict=True): | |
| state.copy_(before) | |
| graph = torch.cuda.CUDAGraph() | |
| with torch.cuda.stream(side): | |
| static_input = torch.empty_like(value) | |
| static_input.copy_(value) | |
| for _ in range(2): | |
| warm_output = operation(static_input) | |
| side.synchronize() | |
| del warm_output | |
| restore(states, saved) | |
| side.synchronize() | |
| graph.capture_begin(capture_error_mode="global") | |
| try: | |
| static_output = operation(static_input) | |
| finally: | |
| graph.capture_end() | |
| side.synchronize() | |
| restore(states, saved) | |
| current.wait_stream(side) | |
| current.synchronize() | |
| assert static_output.dtype == torch.float32 | |
| static_input.record_stream(current) | |
| static_output.record_stream(current) | |
| for state, before in zip(states, saved, strict=True): | |
| assert torch.equal(state.contiguous().view(torch.int16), before.contiguous().view(torch.int16)) | |
| del saved, states | |
| pool = tuple(graph.pool()) | |
| details = describe_pool(torch.cuda.memory_snapshot(mempool_id=pool, include_traces=False), pool) | |
| assert not details["foreign_pool"] and details["reserved_bytes"] > 0 | |
| return {"graph": graph, "static_input": static_input, "static_output": static_output, | |
| "stream": current, "side": side, "pool": pool, | |
| "pool_at_capture": details, | |
| "static_input_storage_bytes": static_input.untyped_storage().nbytes(), | |
| "capture_mutation_restored": True} | |
| def replay_clone(self, entry: dict[str, Any], value: Any) -> Any: | |
| if self.torch.cuda.current_stream(value.device).cuda_stream != entry["stream"].cuda_stream: | |
| raise RuntimeError("Replay stream changed") | |
| entry["static_input"].copy_(value) | |
| entry["graph"].replay() | |
| return entry["static_output"].clone() | |
| def release(self, entry: dict[str, Any]) -> dict[str, Any]: | |
| torch = self.torch | |
| refs = {key: weakref.ref(entry[key]) for key in ("static_input", "static_output", "graph")} | |
| events = [] | |
| def snapshot(stage: str) -> None: | |
| item = describe_pool(torch.cuda.memory_snapshot( | |
| mempool_id=entry["pool"], include_traces=False), entry["pool"]) | |
| item.update(stage=stage, weak_alive={key: ref() is not None for key, ref in refs.items()}) | |
| events.append(item) | |
| entry["stream"].synchronize() | |
| entry["side"].synchronize() | |
| snapshot("before_drop") | |
| entry["graph"].reset() | |
| entry["static_input"] = entry["static_output"] = entry["graph"] = None | |
| snapshot("after_drop") | |
| entry["stream"].synchronize() | |
| entry["side"].synchronize() | |
| with torch.cuda.stream(entry["stream"]): | |
| probe = torch.empty((1,), device=entry["stream"].device, dtype=torch.uint8) | |
| del probe | |
| entry["stream"].synchronize() | |
| snapshot("after_fence_allocator_poll") | |
| final = events[-1] | |
| checks = {"no_graph_refs": not any(final["weak_alive"].values()), | |
| "correct_pool": not final["foreign_pool"], | |
| "pool_active_zero": final["active_bytes"] == 0, | |
| "pool_allocated_zero": final["allocated_bytes"] == 0, | |
| "no_active_blocks": final["active_allocated_blocks"] == 0, | |
| "no_pending_blocks": final["pending_blocks"] == 0, | |
| "known_block_states": not final["unknown_block_states"]} | |
| return {"passed": all(checks.values()), "checks": checks, "events": events} | |