Image-Text-to-Text
MLX
Safetensors
qwen3_5
clairic
cloudflare
systemone
structured-output
information-extraction
multimodal
custom-code
conversational
4-bit precision
Instructions to use mlx-community/clairic-flash-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use mlx-community/clairic-flash-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("mlx-community/clairic-flash-4bit") config = load_config("mlx-community/clairic-flash-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use mlx-community/clairic-flash-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-flash-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "mlx-community/clairic-flash-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use mlx-community/clairic-flash-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-flash-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default mlx-community/clairic-flash-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use mlx-community/clairic-flash-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-flash-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "mlx-community/clairic-flash-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download clairic_runtime/backend.py from mlx-community/clairic-flash-4bit: direct link, hf CLI and curl.
- Browser
- Download file 72.4 kB
-
https://huggingface.co/mlx-community/clairic-flash-4bit/resolve/main/clairic_runtime/backend.py
- Command line
-
hf download hf://mlx-community/clairic-flash-4bit/clairic_runtime/backend.py
-
curl -L -o backend.py https://huggingface.co/mlx-community/clairic-flash-4bit/resolve/main/clairic_runtime/backend.py
72.4 kB
# Modified from the Cloudflare Clairic release (Apache-2.0) for the MLX port:
# - `media_payload` keeps pixel values as NumPy arrays (rounded through bfloat16, as upstream does) and adds
# a content key for the engine's per-request image-feature cache;
# - `load` builds the MLX engine (`mlx_engine.py`) and the torch-free processor (`processor.py`) instead of
# the PyTorch engine and the Transformers processor, and takes engine options instead of device / dtype.
# Everything else is unchanged.
from __future__ import annotations
from collections.abc import Callable, Collection, Sequence
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .api import ClairImage
from .documents import ClairDocument
from PIL import Image
import asyncio
import concurrent.futures
import json
import logging
import math
import uuid
from decimal import Decimal
from pathlib import Path
from typing import Any, cast
from .api import MAX_IMAGES, PredictOutput, PredictRequest
from .errors import ClairInferenceError, ClairQueueFull, ClairRequestError
from .settings import RuntimeSettings
from . import (
MAX_OUTPUT_TOKENS,
ar,
batch_completion,
contracts,
entity_scope,
financial_scope,
grounding,
identifier_scope,
identifier_spelling,
image_financial_lines,
image_focus,
image_identifier,
image_party_names,
image_ragged_rows,
image_rows,
image_table_rows,
ordinary_list_completion,
pdf_number_focus,
printed_numbers,
printed_temporal,
receipt_totals,
row_omission,
source_dates,
source_rows,
table_rows,
)
from . import lists as extraction
from . import model as cm
from . import schema as zs
from .media_cache import ImageCache
from .media import batch_image_data, media_warmup_groups
if TYPE_CHECKING:
MediaPayload = list[dict[str, Any]]
ARTask = tuple[str, list[int], MediaPayload | None]
EncodedRequest = tuple[
dict[str, Any], dict[str, list[str]], list[ARTask], cm.EncodedRecord | None, int
]
logger = logging.getLogger(__name__)
def common_prefix(tasks: Sequence[ARTask]) -> list[int]:
(first, last) = (min((t[1] for t in tasks)), max((t[1] for t in tasks)))
length = next(
(i for (i, (a, b)) in enumerate(zip(first, last)) if a != b),
min(len(first), len(last)),
)
return first[:length]
def expand_request(
request: PredictRequest,
*,
date_role_guidance: bool = True,
source_row_eligibility: bool = False,
local_scoped_pairs: bool = False,
fee_guidance: bool = False,
service_guidance: bool = False,
literal_payment_rows: bool = False,
) -> tuple[dict[str, Any], dict[str, list[str]]]:
contracts.validate_public_fields(request.fields)
(fields, forecasts) = zs.expand_forecasts(
financial_scope.prepare(
request.fields,
has_media=bool(request.images or request.documents),
fee_guidance=fee_guidance,
service_guidance=service_guidance,
)
)
zs.validate_fields(fields)
zs.validate_reference_date(request.reference_date)
fields = {n: dict(s) for (n, s) in fields.items()}
for spec in fields.values():
if date_role_guidance and spec["type"] == "date":
instructions = spec.get("instructions", "")
spec["instructions"] = (
(instructions + "\n" if instructions else "")
+ "Follow field instructions. Match the requested date role; null if unavailable."
)
annotations: dict[str, dict[str, Any]] = {}
row_inputs: dict[str, str]
row_names: dict[str, str]
(row_inputs, row_names) = ({}, {})
source_tables = table_rows.prepare(
request.state,
fields,
has_media=bool(request.images or request.documents),
allow_scoped_pairs=source_row_eligibility or local_scoped_pairs,
)
for name, spec in list(fields.items()):
if spec["type"] != "list":
continue
source_candidates = source_tables.get(name)
row_eligibility = source_row_eligibility or any(
(row.get("eligibility_only") for row in source_candidates or [])
)
budget = (
len(source_candidates)
if source_candidates
else zs.MAX_LIST_ITEMS + 1
if (request.images or request.documents) and "max_items" not in spec
else extraction.list_query_budget(request.state, spec)[0]
)
queries = []
item_spec = contracts.item_spec(spec)
for index in range(1, budget + 1):
child = zs.list_item_name(name, index)
if child in fields:
raise ValueError("Generated list field conflicts with " + child)
fields[child] = {
k: v
for (k, v) in spec.items()
if k not in {"type", "items", "instructions", "max_items"}
}
fields[child].update(item_spec)
fields[child].update(
type=item_spec["type"],
instructions=(
f"Item {index} of {name}, following requested order and duplicate policy. "
+ spec.get("instructions", "")
+ " Return null if this item does not exist."
).strip(),
)
if source_candidates:
fields[child] = table_rows.row_spec(
spec, source_candidates[index - 1], eligibility=row_eligibility
)
if item_spec["type"] in ("text", "number", "object") and (
not (source_candidates and row_eligibility)
):
fields[child]["nullable"] = True
if "enum" in fields[child] and None not in fields[child]["enum"]:
fields[child]["enum"] = [*fields[child]["enum"], None]
annotations[child] = {"parent": name, "index": index}
queries.append(child)
annotations[name] = (
{
"source_fields": queries,
"source_rows": source_candidates,
"source_row_eligibility": row_eligibility,
}
if source_candidates
else {"query_fields": queries}
)
if local_scoped_pairs and source_candidates:
(inputs, names) = table_rows.local_scope_context(
source_candidates, queries, offset=len(row_names)
)
row_inputs.update(inputs)
row_names.update(names)
(fields, temporal_kinds) = printed_temporal.prepare(fields)
payments = (
printed_numbers.payment_lists(request.fields)
if literal_payment_rows and (request.images or request.documents)
else []
)
payment_rows = [
child
for parent in payments
for child in annotations.get(parent, {}).get("query_fields", ())
]
(fields, number_contracts) = printed_numbers.prepare(
fields, payment_rows=payment_rows
)
meta = {
"category": "extraction",
"construction": zs.EVIDENCE_CONSTRUCTION,
"extraction_contract": {"fields": annotations},
"printed_temporal": temporal_kinds,
"printed_numbers": number_contracts,
}
return (
{
"input": request.state,
"fields": fields,
"reference_date": request.reference_date,
"_meta": meta,
"_source_row_inputs": row_inputs,
"_source_row_names": row_names,
"_source_row_gates": [
query
for annotation in annotations.values()
if annotation.get("source_row_eligibility")
for query in annotation["source_fields"]
],
},
forecasts,
)
def typed_readout_record(
request: PredictRequest,
record: dict[str, Any],
*,
automatic_temporal: dict[str, dict[str, Any]] | None = None,
**expand_options: Any,
) -> dict[str, Any] | None:
typed = ar.typed_record(record)
if typed is None or not automatic_temporal:
return typed
original = request.model_copy(
update={"fields": {**request.fields, **automatic_temporal}}
)
(original_record, _) = expand_request(original, **expand_options)
return ar.typed_record(original_record)
def scalar_result(
result: dict[str, Any],
spec: dict[str, Any],
tokenizer: Any,
source: str | None = None,
evidence: grounding.Source | None = None,
*,
excluded_token_id: int | None = None,
provenance: grounding.Source | None = None,
) -> dict[str, Any]:
from jsonschema import Draft202012Validator, ValidationError
meta = result.get("meta_info", {})
finish = meta.get("finish_reason", {}) or {}
eos = tokenizer.convert_tokens_to_ids(ar.EOS)
complete = finish.get("type") == "stop" and finish.get("matched") == eos
ids = result.get("output_ids")
if ids is not None:
ids = ids[:-1] if ids and ids[-1] == eos else ids
raw = tokenizer.decode(
ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
)
else:
raw = result.get("text", "")
if complete and raw.endswith(ar.EOS):
raw = raw[: -len(ar.EOS)]
raw = raw.strip()
value = None
valid = False
span = None
failure_reason = None
if complete:
try:
value = json.loads(
raw, parse_constant=lambda s: (_ for _ in ()).throw(ValueError(s))
)
Draft202012Validator(ar.schema(spec)).validate(value)
if isinstance(value, float) and (not math.isfinite(value)):
raise ValueError("Nonfinite number")
if value is not None and evidence is not None:
literal = json.loads(raw, parse_float=Decimal, parse_int=Decimal)
(found, span) = (
evidence.span(literal, spec=spec)
if getattr(evidence, "normalize_whitespace", False)
else evidence.span(literal)
)
if not found:
failure_reason = "not_in_source"
raise ValueError("AR extraction is not a literal source value")
decimals = zs.allowed_decimals(spec) if spec["type"] == "number" else None
if value is not None and decimals is not None:
literal = json.loads(raw, parse_float=Decimal, parse_int=Decimal)
value = ar.normalize_number(literal, spec)
Draft202012Validator(ar.schema(spec)).validate(value)
if value is not None and spec["type"] == "object":
precise = json.loads(raw, parse_float=Decimal, parse_int=Decimal)
value = {
name: ar.normalize_number(precise[name], child)
if child["type"] == "number"
else value[name]
for (name, child) in spec["properties"].items()
}
Draft202012Validator(ar.schema(spec)).validate(value)
valid = True
except (ValueError, TypeError, ValidationError):
value = None
prediction = {
"value": value,
"valid": valid,
"complete": complete,
"decoder": ar.VERSION,
"tokens": int(meta.get("completion_tokens", 0)),
"raw": raw,
"span": span if valid else None,
}
if failure_reason is not None:
prediction["failure_reason"] = failure_reason
if valid and evidence is not None and (value is not None):
prediction["grounding_method"] = "literal_source"
if span is not None:
prediction["span_method"] = "unique_literal_match"
if (
getattr(evidence, "normalize_whitespace", False)
and isinstance(value, str)
and (value not in evidence.text)
):
prediction["grounding_method"] = "whitespace_normalized_source"
if span is not None:
prediction["span_method"] = "unique_whitespace_normalized_match"
token_logprobs = meta.get("output_token_logprobs")
null_scores = meta.get("output_token_ids_logprobs")
if excluded_token_id is not None:
score = (
next(
(
float(token[0])
for token in null_scores[0]
if token[1] == excluded_token_id
),
None,
)
if null_scores
else None
)
if score is None or math.isnan(score) or score > 1e-06:
raise ValueError(
"Transformers omitted a valid row-exclusion log probability"
)
prediction.update(
p_excluded=math.exp(min(0.0, score)),
p_excluded_method="ar_first_token_false_probability",
)
if not spec.get("nullable", True):
prediction.update(p_null=0.0, p_null_method="schema_disallows_null")
elif null_scores:
null_id = tokenizer.convert_tokens_to_ids("null")
score = next(
(float(token[0]) for token in null_scores[0] if token[1] == null_id), None
)
if score is None or math.isnan(score) or score > 1e-06:
raise ValueError("Transformers omitted a valid null-token log probability")
prediction.update(
p_null=math.exp(min(0.0, score)),
p_null_method="ar_first_token_null_probability",
)
if valid and token_logprobs:
log_probability = sum((float(token[0]) for token in token_logprobs))
if not math.isfinite(log_probability) or log_probability > 1e-06:
raise ValueError("Transformers returned invalid token log probabilities")
prediction.update(
confidence=math.exp(min(0.0, log_probability)),
log_probability=log_probability,
confidence_method="ar_sequence_probability",
)
elif not valid:
prediction["confidence"] = 0.0
if (
valid
and evidence is None
and (provenance is not None)
and (value is not None)
and (spec["type"] == "number")
):
literal = json.loads(raw, parse_float=Decimal, parse_int=Decimal)
(found, literal_span) = provenance.span(literal)
if found and literal_span is not None:
prediction.update(span=literal_span, span_method="unique_literal_match")
if valid and evidence is None and isinstance(value, str) and value and source:
start = source.find(value)
if start >= 0 and source.find(value, start + 1) < 0:
prediction.update(
span=[start, start + len(value)], span_method="unique_literal_match"
)
return prediction
def validate_eos(path: Path, tokenizer: Any) -> None:
config = json.loads((path / "config.json").read_text())
generation = json.loads((path / "generation_config.json").read_text())
eos = tokenizer.convert_tokens_to_ids(ar.EOS)
configured = [
config.get("eos_token_id"),
config.get("text_config", {}).get("eos_token_id"),
generation.get("eos_token_id"),
]
if any((value != eos for value in configured)):
raise ValueError(
"Model, text, and generation EOS must match the <|im_end|> terminator"
)
class ClairicFlashModel:
RELEASE_MODEL_ID = "clairic-v2"
COMPACT_LIST_PROMPTS = True
DATE_ROLE_GUIDANCE = True
RECOVER_NULLS = True
SOURCE_ROW_ELIGIBILITY = False
LOCAL_SOURCE_SCOPE = True
FINANCIAL_SERVICE_GUIDANCE = True
FINANCIAL_FEE_GUIDANCE = False
EXPLICIT_TEMPORAL_READS = True
SPELL_VISUAL_PO_REFERENCES = True
FOCUS_SPARSE_IMAGES = True
FOCUS_IMAGE_IDENTIFIERS = True
FOCUS_IMAGE_PARTY_NAMES = True
LITERAL_IMAGE_PAYMENT_ROWS = False
BIND_IMAGE_FINANCIAL_LINES = True
PRESERVE_IMAGE_ENTITY_NAMES = False
BIND_IMAGE_TABLE_ROWS = True
BIND_IMAGE_RAGGED_ROWS = True
IMAGE_TABLE_ROW_SCALE = 2
REQUEST_TIMEOUT_SECONDS = 600
DATE_SUPPLEMENT_TIMEOUT_SECONDS = 600
RECEIPT_SUPPLEMENT_TIMEOUT_SECONDS = 600
def __init__(
self,
engine: Any,
tokenizer: Any,
processor: Any,
settings: RuntimeSettings,
head_config: dict[str, Any],
) -> None:
(self.engine, self.tokenizer, self.processor) = (engine, tokenizer, processor)
(self.settings, self.head_config) = (settings, head_config)
(self.max_concurrency, self.model_id) = (
settings.max_concurrency,
settings.model_id,
)
self._encoder = concurrent.futures.ThreadPoolExecutor(
max_workers=1, thread_name_prefix="clairic-encode"
)
self._in_flight = 0
self._image_cache = ImageCache()
def encode(
self,
request: PredictRequest,
*,
identifier_view: bool = False,
party_name_view: bool = False,
image_views: Callable[[], Sequence[Image.Image]] | None = None,
physical_row_view: bool = False,
image_view_key: str | None = None,
pdf_spelling_view: bool = False,
identifier_spelling_disabled: bool = False,
po_detail_view: bool = False,
) -> EncodedRequest:
original_request = request
automatic_temporal = {}
if self.EXPLICIT_TEMPORAL_READS:
routed_fields = printed_temporal.explicit_fields(request.fields)
automatic_temporal = {
name: spec
for (name, spec) in request.fields.items()
if routed_fields[name] is not spec
}
request = request.model_copy(update={"fields": routed_fields})
if self.PRESERVE_IMAGE_ENTITY_NAMES:
request = request.model_copy(
update={
"fields": entity_scope.prepare(
request.fields,
has_media=bool(request.images or request.documents),
)
}
)
(scoped_fields, query_labels) = identifier_scope.prepare_fields(
request.fields, has_media=bool(request.images or request.documents)
)
if scoped_fields is not request.fields:
request = request.model_copy(update={"fields": scoped_fields})
if self.FOCUS_SPARSE_IMAGES and request.images or pdf_spelling_view:
request = request.model_copy(
update={"fields": image_focus.preserve_scalar_fields(request.fields)}
)
expand_options = {
"date_role_guidance": self.DATE_ROLE_GUIDANCE,
"source_row_eligibility": self.SOURCE_ROW_ELIGIBILITY,
"local_scoped_pairs": self.LOCAL_SOURCE_SCOPE,
"service_guidance": self.FINANCIAL_SERVICE_GUIDANCE,
"fee_guidance": self.FINANCIAL_FEE_GUIDANCE,
"literal_payment_rows": self.LITERAL_IMAGE_PAYMENT_ROWS,
}
(record, forecasts) = expand_request(request, **expand_options)
state_only_fields = {
name
for (name, spec) in original_request.fields.items()
if grounding.state_only(spec)
}
if self.SPELL_VISUAL_PO_REFERENCES and (not identifier_spelling_disabled):
identifier_spelling.prepare(
record,
original_request.fields,
query_labels,
has_media=bool(original_request.images or original_request.documents),
)
if query_labels:
spelling = record["_meta"].get("identifier_spelling", {})
record.setdefault("_source_row_names", {}).update(
{
name: name if name in spelling else label
for (name, label) in query_labels.items()
}
)
if physical_row_view:
for spec in record["fields"].values():
spec["_physical_row_scope"] = True
image_rows.prepare(
record,
has_media=bool(request.images or request.documents),
share_description=getattr(
self._image_cache, "_share_image_row_description", False
),
)
record["_list_source_bounds"] = source_rows.list_bounds(
request.state,
request.fields,
has_images=bool(request.images or request.documents),
)
sources: tuple[ClairImage | ClairDocument, ...] = (
*request.images,
*request.documents,
)
source_key = ImageCache.key(sources) if sources else None
image_key = source_key
if image_key is not None:
image_key += (
":identifier:" + image_identifier.VIEW_ID
if identifier_view
else ":default"
)
if image_view_key is not None:
image_key += ":view:" + image_view_key
prepared = self._image_cache.get(image_key) if image_key is not None else None
if sources and prepared is None:
images = (
list(image_views())
if image_views is not None
else [i.to_pil() for i in request.images]
)
original_images = list(images)
if self.FOCUS_SPARSE_IMAGES or pdf_spelling_view or po_detail_view:
focus = (
(
lambda image: image_focus.focus_document(
image,
max_pixels=identifier_spelling.MANY_IMAGE_MAX_PIXELS,
max_scale=3.0,
)
)
if po_detail_view
else image_focus.focus_document
)
unfocused_image = None
if (
type(self) is ClairicFlashModel
and identifier_view
and (len(request.images) == 1)
and (not request.documents)
):
default = self._image_cache.entries.get(
cast(str, source_key) + ":default"
)
if (
default is not None
and len(default.images) == 1
and default.source_images
and (default.images[0] is default.source_images[0])
):
unfocused_image = default.images[0]
if unfocused_image is None:
images = [focus(image) for image in images]
else:
images = [
image if image is unfocused_image else focus(image)
for image in images
]
if image_views is None:
for document in request.documents:
pages = document.render(MAX_IMAGES - len(images))
images.extend(pages)
original_images.extend(pages)
media: cm.EncodedMedia | None = cm.encode_media(
self.tokenizer, images, self.processor
)
prepared = self._image_cache.put(
cast(str, image_key),
images,
cast(cm.EncodedMedia, media),
self.media_payload(
request,
cast(dict[str, Any], cast(cm.EncodedMedia, media).media),
source_digest=image_key,
feature_namespace=self._image_cache.namespace,
),
source_images=original_images,
)
images = prepared.images if prepared else []
media = prepared.media if prepared else None
payload = prepared.payload if prepared else None
record["_media_payload"] = payload
record["_original_images"] = prepared.source_images if prepared else []
tasks = []
state_tasks = set()
eos = self.tokenizer.convert_tokens_to_ids(ar.EOS)
if images:
names = ar.names(record)
if names:
marker = "PARALLEL_AR_CONTENT_4A61DB"
messages = [
{
"role": "user",
"content": [{"type": "image", "image": i} for i in images]
+ [{"type": "text", "text": marker}],
}
]
template = self.processor.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
(header, footer) = template.split(marker)
boundary = header.rfind("<|vision_end|>")
if boundary < 0:
raise ValueError(
"The image chat template is missing its vision terminator"
)
boundary += len("<|vision_end|>")
replacements = [
self.processor.replace_image_token(
cast(cm.EncodedMedia, media).media, i
)
for i in range(len(images))
]
(prefix_text, _) = self.processor.get_text_with_replacements(
[header[:boundary]], replacements, [], []
)
prefix = self.tokenizer(prefix_text, add_special_tokens=False)[
"input_ids"
][0]
source = self.tokenizer.encode(
request.state, add_special_tokens=False, split_special_tokens=True
)
scoped_media = {
name
for (name, spec) in original_request.fields.items()
if grounding.explicit_media_scope(spec)
}
tails = self.tokenizer(
[
header[boundary:]
+ (
ar.MEDIA_EVIDENCE_INSTRUCTION + "\n\n"
if request.state and name in scoped_media
else ""
)
+ ar.instruction(
record, name, compact_lists=self.COMPACT_LIST_PROMPTS
).rstrip()
+ (
(
"\n\nAccompanying text:\n"
if name in scoped_media
else "\n\nDocument:\n"
)
if request.state
else footer
)
for name in names
],
add_special_tokens=False,
)["input_ids"]
if request.state:
footer_ids = self.tokenizer.encode(footer, add_special_tokens=False)
tails = [tail + source + footer_ids for tail in tails]
tasks = [
(name, prefix + tail, payload)
for (name, tail) in zip(names, tails, strict=True)
]
state_tasks = {
name
for name in names
if name in state_only_fields
or grounding.state_only(record["fields"][name])
}
if state_tasks:
text_template = self.tokenizer.apply_chat_template(
[{"role": "user", "content": marker}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
(text_header, text_footer) = text_template.split(marker)
text_footer_ids = self.tokenizer.encode(
text_footer, add_special_tokens=False
)
tasks = [
(
name,
self.tokenizer.encode(
text_header
+ ar.instruction(
record,
name,
compact_lists=self.COMPACT_LIST_PROMPTS,
)
+ "\n\nDocument:\n",
add_special_tokens=False,
)
+ source
+ text_footer_ids,
None,
)
if name in state_tasks
else (name, ids, task_media)
for (name, ids, task_media) in tasks
]
else:
marker = "PARALLEL_AR_CONTENT_4A61DB"
template = self.tokenizer.apply_chat_template(
[{"role": "user", "content": marker}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
(header, footer) = template.split(marker)
source = self.tokenizer.encode(
request.state, add_special_tokens=False, split_special_tokens=True
)
tail = self.tokenizer.encode(footer, add_special_tokens=False)
for name in ar.names(record):
ids = (
self.tokenizer.encode(
header
+ ar.instruction(
record, name, compact_lists=self.COMPACT_LIST_PROMPTS
)
+ "\n\nDocument:\n",
add_special_tokens=False,
)
+ (
self.tokenizer.encode(
record["_source_row_inputs"][name],
add_special_tokens=False,
split_special_tokens=True,
)
if name in record["_source_row_inputs"]
else source
)
+ tail
)
tasks.append((name, ids, None))
typed = typed_readout_record(
request, record, automatic_temporal=automatic_temporal, **expand_options
)
item = None
if typed:
item = cm.encode_record(
self.tokenizer,
typed,
self.settings.max_length,
media=media,
answer_slots=True,
text_answer_slots=True,
)
if item is None or item.text != record["input"]:
raise ValueError("Typed fields and input must fit one complete context")
if any((len(ids) > self.settings.max_length for (_, ids, _) in tasks)):
raise ValueError(
"AR input exceeds the model context; no input was truncated"
)
record["_ar_tasks"] = tasks
source = (
grounding.Source(request.state, normalize_whitespace=True)
if tasks and (not images or state_tasks)
else None
)
record["_ar_source"] = source if not images else None
record["_ar_sources"] = dict.fromkeys(state_tasks, source)
record["_ar_constraints"] = {
name: source.constraint(record["fields"][name])
if (source := record["_ar_sources"].get(name, record["_ar_source"]))
is not None
and grounding.applies(record["fields"][name], media)
else {"json_schema": json.dumps(ar.schema(record["fields"][name]))}
for (name, _, media) in tasks
}
record["_cache_salt"] = prepared.salt if prepared else uuid.uuid4().hex
if state_tasks:
record["_ar_task_salts"] = {name: uuid.uuid4().hex for name in state_tasks}
if len(tasks) > 1:
prefix = common_prefix(tasks)
if len(prefix) >= 128:
record["_ar_prefix"] = prefix
groups: dict[str, list[tuple[str, list[int], Any]]] = {}
annotations = record["_meta"]["extraction_contract"]["fields"]
for task in tasks:
parent = annotations.get(task[0], {}).get("parent")
if parent:
groups.setdefault(parent, []).append(task)
record["_ar_group_prefixes"] = [
common_prefix(group) for group in groups.values() if len(group) > 1
]
encoded = (record, forecasts, tasks, item, eos)
detail_view = (
identifier_view
or party_name_view
or physical_row_view
or identifier_spelling_disabled
)
if self.BIND_IMAGE_TABLE_ROWS and (not detail_view):
encoded = image_table_rows.prepare(
request, encoded, self.encode, scale=self.IMAGE_TABLE_ROW_SCALE
)
if self.BIND_IMAGE_RAGGED_ROWS and (not detail_view):
encoded = image_ragged_rows.prepare(
request, encoded, self.encode, scale=self.IMAGE_TABLE_ROW_SCALE
)
if self.BIND_IMAGE_FINANCIAL_LINES and (not detail_view):
encoded = image_financial_lines.prepare(request, encoded, self.encode)
if self.FOCUS_IMAGE_IDENTIFIERS and (not detail_view):
prefix_overlap = getattr(self, "_invoice_prefix_overlap", None)
if prefix_overlap is not None:
prefix_overlap.start(original_request, encoded, eos)
unfocused_image = None
if (
type(self) is ClairicFlashModel
and self.FOCUS_SPARSE_IMAGES
and (len(request.images) == 1)
and (not request.documents)
and (prepared is not None)
and (len(prepared.images) == 1)
and prepared.source_images
and (prepared.images[0] is prepared.source_images[0])
):
unfocused_image = prepared.images[0]
if unfocused_image is None:
encoded = image_identifier.prepare(
original_request, encoded, self.encode
)
else:
encoded = image_identifier.prepare(
original_request,
encoded,
self.encode,
unfocused_image=unfocused_image,
)
if self.FOCUS_IMAGE_PARTY_NAMES and (not detail_view):
encoded = self._party_name_tasks(original_request, encoded)
if not detail_view:
encoded = pdf_number_focus.prepare(
original_request, encoded, self.encode, enabled=self.FOCUS_SPARSE_IMAGES
)
encoded = identifier_spelling.prepare_pdf_tasks(
original_request, encoded, self.encode, enabled=True
)
encoded = identifier_spelling.prepare_many_image_tasks(
original_request, encoded, self.encode
)
encoded = identifier_spelling.prepare_fallback(
original_request, encoded, self.encode
)
record = encoded[0]
record["_first_item_list_gating"] = getattr(
self._image_cache, "_share_image_row_description", False
)
if record.get("_ar_task_salts"):
record["_ar_media_groups"] = media_warmup_groups(
record["_ar_tasks"],
default_salt=record["_cache_salt"],
task_salts=record["_ar_task_salts"],
annotations=record["_meta"]["extraction_contract"]["fields"],
)
return encoded
def _party_name_tasks(
self, request: PredictRequest, encoded: EncodedRequest
) -> EncodedRequest:
selected = image_party_names.fields(request.fields)
if not selected or len(request.images) != 1 or request.documents:
return encoded
(record, forecasts, tasks, typed, eos) = encoded
originals = record.get("_original_images")
image = originals[0] if originals else request.images[0].to_pil()
views = image_party_names.views(image)
if len(views) == 1:
return encoded
fields = {
name: {
**request.fields[name],
"instructions": request.fields[name].get("instructions", "")
+ image_party_names.VIEW_INSTRUCTION,
}
for name in selected
}
child_request = request.model_copy(update={"fields": fields})
(child, _, replacements, _, _) = self.encode(
child_request,
party_name_view=True,
image_views=lambda: views,
image_view_key=image_party_names.VIEW_ID,
)
by_name = {task[0]: task for task in replacements}
tasks = [by_name.get(task[0], task) for task in tasks]
salts = dict(record.get("_ar_task_salts", {}))
for name in selected:
record["fields"][name] = child["fields"][name]
record["_ar_constraints"][name] = child["_ar_constraints"][name]
salts[name] = child["_cache_salt"]
record.update(
_ar_tasks=tasks, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[]
)
return (record, forecasts, tasks, typed, eos)
@staticmethod
def media_payload(
request: PredictRequest,
encoded: dict[str, Any],
*,
source_digest: str | None = None,
feature_namespace: str = "",
) -> list[dict[str, Any]]:
# MLX port: pixel values stay NumPy, rounded through bfloat16 as upstream does before the model sees
# them, and carry a content key so the engine can reuse the vision features within a request.
import hashlib
import mlx.core as mx
import numpy as np
digest = source_digest or ImageCache.key((*request.images, *request.documents))
pixels = np.array(mx.array(np.asarray(encoded["pixel_values"], dtype=np.float32)).astype(mx.bfloat16).astype(mx.float32))
grid = np.asarray(encoded["image_grid_thw"])
return [
{
"format": "processor_output",
"clair_media_digest": digest,
"clair_pixels_key": hashlib.sha1(pixels.tobytes() + grid.tobytes()).hexdigest(),
"pixel_values": pixels,
"image_grid_thw": grid,
}
]
async def _ar(
self, record: dict[str, Any], task: ARTask, eos: int
) -> tuple[str, dict[str, Any]]:
if "_ar_future" not in record:
record["_ar_future"] = asyncio.create_task(self._ar_all(record, eos))
predictions = await record["_ar_future"]
return (task[0], predictions[task[0]])
async def _ar_all(
self, record: dict[str, Any], eos: int
) -> dict[str, dict[str, Any]]:
tasks = record["_ar_tasks"]
salt = record["_cache_salt"]
row_gates = set(record.get("_source_row_gates", ()))
false_id = self.tokenizer.convert_tokens_to_ids("false") if row_gates else None
score_ids = [self.tokenizer.convert_tokens_to_ids("null")]
if false_id is not None:
score_ids.append(false_id)
results = await ordinary_list_completion.generate(
self.engine,
record,
tasks,
eos,
self.tokenizer,
initial_frontier=32
if type(self) is ClairicFlashModel
and self.settings.model_id == "clairic-flash"
and (self.max_concurrency == self.settings.max_concurrency == 1)
else 0,
next_frontier=64,
parse_result=scalar_result,
normalize=lambda record, predictions: printed_numbers.normalize_predictions(
record, printed_temporal.normalize_predictions(record, predictions)
),
input_ids=[ids for (_, ids, _) in tasks],
image_data=batch_image_data([media for (_, _, media) in tasks]),
rid=["ar-" + uuid.uuid4().hex for _ in tasks],
cache_salt=[
record.get("_ar_task_salts", {}).get(name, salt)
for (name, _, _) in tasks
],
return_logprob=True,
logprob_start_len=-1,
token_ids_logprob=score_ids,
sampling_params=[
{
"temperature": 0,
"max_new_tokens": image_rows.output_limit(
record, name, len(ids), self.settings.max_length
)
if image_rows.is_inventory(record["fields"][name])
else record.get("_ar_max_output_tokens", MAX_OUTPUT_TOKENS),
"stop_token_ids": [eos],
**(
record["_ar_constraints"][name]
if name in record.get("_ar_constraints", {})
else {
"json_schema": json.dumps(ar.schema(record["fields"][name]))
}
),
"skip_special_tokens": False,
}
for (name, ids, _) in tasks
],
)
if usage := record.get("_completed_generation_usage"):
usage["tasks"].extend(tasks)
usage["readings"].extend(
(
{
"tokens": int(
result.get("meta_info", {}).get("completion_tokens", 0)
)
}
for result in results
)
)
predictions = {
name: batch_completion.unused_prediction(result)
if result.get("_clair_unused_row_proposal")
else scalar_result(
result,
record["fields"][name],
self.tokenizer,
record["input"],
record.get("_ar_sources", {}).get(name, record.get("_ar_source"))
if grounding.applies(record["fields"][name], media)
else None,
excluded_token_id=false_id if name in row_gates else None,
provenance=record.get("_ar_sources", {}).get(
name, record.get("_ar_source")
)
if media is None and (not record["fields"][name].get("knowledge"))
else None,
)
for ((name, _, media), result) in zip(tasks, results, strict=True)
}
if any(
(
"confidence" not in prediction
for prediction in predictions.values()
if not prediction.get("_unused_row_proposal")
)
):
raise ClairInferenceError(
"Transformers omitted generation log probabilities"
)
if any(
(
"p_null" not in prediction
for prediction in predictions.values()
if not prediction.get("_unused_row_proposal")
)
):
raise ClairInferenceError(
"Transformers omitted null-token log probabilities"
)
if self.RECOVER_NULLS and (not record.get("_null_recovery")):
fields = {
name
for (name, prediction) in predictions.items()
if (
prediction["valid"]
and prediction["value"] is None
or prediction.get("failure_reason") == "not_in_source"
)
and (not record["fields"][name].get("knowledge"))
and (name not in record.get("_meta", {}).get("printed_temporal", {}))
and (name not in record.get("_meta", {}).get("printed_numbers", {}))
and (
record["fields"][name]["type"] == "number"
or (record["fields"][name]["type"] == "text" and record["input"])
)
and (
not record["_meta"]["extraction_contract"]["fields"]
.get(name, {})
.get("parent")
)
}
if fields:
repair = await asyncio.get_running_loop().run_in_executor(
self._encoder, self.null_recovery_record, record, fields
)
if repair["_ar_tasks"]:
alternatives = await self._ar_all(repair, eos)
retry_lengths = {
name: len(ids) for (name, ids, _) in repair["_ar_tasks"]
}
for name, alternative in alternatives.items():
original = predictions[name]
tokens = original["tokens"] + alternative["tokens"]
if (
alternative["valid"]
and alternative["value"] is not None
and (
alternative.get("grounding_method")
in ("literal_source", "whitespace_normalized_source")
or record["fields"][name]["type"] == "number"
or (
alternative.get("span") is not None
and alternative.get("confidence", 0.0)
> alternative.get("p_null", 1.0)
)
)
):
predictions[name] = {**alternative, "null_recovered": True}
predictions[name]["tokens"] = tokens
predictions[name]["retry_input_tokens"] = retry_lengths[name]
if plan := row_omission.prepare(record, predictions):
repair, _ = plan
alternatives = await self._ar_all(repair, eos)
predictions, _ = row_omission.merge(record, predictions, alternatives, plan)
return predictions
def null_recovery_record(
self, record: dict[str, Any], fields: Collection[str]
) -> dict[str, Any]:
original = self.tokenizer.encode(
ar.EXTRACTION_INSTRUCTION, add_special_tokens=False
)
tasks = []
for name, ids, media in record["_ar_tasks"]:
if name not in fields:
continue
source = record.get("_ar_sources", {}).get(name, record.get("_ar_source"))
instruction = (
ar.COMPUTATION_INSTRUCTION
if source is not None
else ar.RECOVERY_INSTRUCTION
)
replacement = self.tokenizer.encode(instruction, add_special_tokens=False)
matches = [
i
for (i, token) in enumerate(ids)
if token == original[0] and ids[i : i + len(original)] == original
]
if len(matches) != 1:
continue
start = matches[0]
prompt = ids[:start] + replacement + ids[start + len(original) :]
if len(prompt) <= self.settings.max_length:
tasks.append((name, prompt, media))
return {
**record,
"_ar_tasks": tasks,
"_ar_prefix": None,
"_ar_group_prefixes": [],
"_ar_media_groups": [],
"_null_recovery": True,
}
async def _typed(
self,
request: PredictRequest,
item: cm.EncodedRecord,
media: list[dict[str, Any]] | None = None,
) -> dict[str, dict[str, Any]]:
if (request.images or request.documents) and media is None:
media = self.media_payload(request, cast(dict[str, Any], item.media))
return await self.engine.read_typed(item, media)
def _encode_receipt_repairs(
self,
request: PredictRequest,
record: dict[str, Any],
predictions: dict[str, dict[str, Any]],
) -> list[tuple[str, dict[str, Any], dict[str, Any]]]:
prepared = []
value: Any
p_null: Any
source = {**record, "input": request.state, "fields": request.fields}
for name in record.get("_receipt_sum_fields", ()):
previous = predictions.get(name, {})
(value, p_null) = (previous.get("value"), previous.get("p_null"))
if (
type(value) not in (int, float)
or not math.isfinite(value)
or previous.get("decoder") != ar.VERSION
or (type(p_null) not in (int, float))
or (not 0 <= p_null <= 1)
):
continue
packet = receipt_totals.prepare_task(source, name)
if packet is None:
continue
tasks = []
for task in packet["tasks"]:
rendered = self.tokenizer.apply_chat_template(
[{"role": "user", "content": task["prompt"]}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
ids = self.tokenizer.encode(rendered, add_special_tokens=False)
if not ids or len(ids) > self.settings.max_length:
raise ValueError(
"Receipt selector exceeds the model context; no input was truncated"
)
tasks.append((task["occurrence_id"], ids, None))
auxiliary = {
"input": request.state,
"fields": {
key: {"type": "boolean", "nullable": True} for (key, _, _) in tasks
},
"_ar_tasks": tasks,
"_ar_constraints": {
task["occurrence_id"]: task["grammar"] for task in packet["tasks"]
},
"_ar_prefix": common_prefix(tasks),
"_cache_salt": uuid.uuid4().hex,
"_null_recovery": True,
"_ar_max_output_tokens": 8,
"_meta": {"extraction_contract": {"fields": {}}},
}
prepared.append((name, packet, auxiliary))
return prepared
async def _complete_receipt_sums(
self,
request: PredictRequest,
record: dict[str, Any],
predictions: dict[str, dict[str, Any]],
eos: int,
progress: dict[str, list[Any]] | None = None,
) -> tuple[
dict[str, dict[str, Any]],
list[tuple[str, list[int], Any]],
list[dict[str, Any]],
]:
if progress is None:
progress = {"encodings": [], "tasks": [], "readings": []}
encoding = asyncio.get_running_loop().run_in_executor(
self._encoder, self._encode_receipt_repairs, request, record, predictions
)
progress["encodings"].append(encoding)
prepared = await asyncio.shield(encoding)
updates = {}
for name, packet, auxiliary in prepared:
decisions = {}
tasks = auxiliary["_ar_tasks"]
for start in range(0, len(tasks), 32):
batch = {
**auxiliary,
"_ar_tasks": tasks[start : start + 32],
"_ar_prefix": auxiliary["_ar_prefix"] if start == 0 else None,
"_completed_generation_usage": progress,
}
completed = len(progress["tasks"])
results = await self._ar_all(batch, eos)
if len(progress["tasks"]) == completed:
progress["tasks"].extend(batch["_ar_tasks"])
progress["readings"].extend(results.values())
decisions.update(
{
key: {**answer, "raw_json": answer["raw"]}
for (key, answer) in results.items()
}
)
checked = receipt_totals.validate_result(packet, decisions)
if (
not checked["valid"]
or checked["value"] is None
or (not checked.get("selection_scores"))
or (checked["value"] == predictions[name]["value"])
):
continue
updates[name] = {
"value": checked["value"],
"valid": True,
"complete": True,
"decoder": "source_receipt_block_sum_v1",
"span": None,
"raw": checked["decimal_sum"],
**checked["selection_scores"],
"p_null": predictions[name]["p_null"],
"p_null_method": "primary_numeric_ar_first_token_null_probability",
"source_receipt_operands": [
{key: item[key] for key in ("span", "printed")}
for item in checked["operands"]
],
"tokens": sum((answer["tokens"] for answer in decisions.values())),
}
return (updates, progress["tasks"], progress["readings"])
def _encode_request(self, request: PredictRequest) -> EncodedRequest:
source: Any
previous = self._image_cache
cache = ImageCache(
max_bytes=previous.max_bytes, max_entries=previous.max_entries
)
cache._share_image_row_description = (
type(self) is ClairicFlashModel
and len(getattr(request, "images", ())) == 1
and (not getattr(request, "documents", ()))
)
self._image_cache = cache
try:
encoded = self.encode(request)
fields = {
name: spec
for (name, spec) in getattr(request, "fields", {}).items()
if source_dates.eligible(spec)
}
if fields and (not request.images) and (not request.documents):
record = encoded[0]
fields = {
name: spec
for (name, spec) in fields.items()
if name not in record.get("_meta", {}).get("printed_temporal", {})
}
source = source_dates.DateSource(request.state)
names = [name for (name, spec) in fields.items() if source.allows(spec)]
if names:
internal_fields = {
name: {
"type": "text",
"nullable": fields[name].get("nullable", True),
"instructions": source_dates.INSTRUCTION
+ "\n"
+ fields[name].get("instructions", ""),
}
for name in names
}
internal = request.model_copy(update={"fields": internal_fields})
try:
(dated, _, _, _, date_eos) = self.encode(internal)
except ValueError:
return encoded
dated["_ar_constraints"].update(
{
name: source.constraint(
nullable=internal_fields[name]["nullable"]
)
for name in names
}
)
dated.update(
_ar_source=source,
_ar_sources=dict.fromkeys(names, source),
_null_recovery=True,
)
record["_date_supplement"] = (dated, source, names, date_eos)
if (
type(self) is ClairicFlashModel
and (not getattr(request, "images", ()))
and (not getattr(request, "documents", ()))
):
source = {
"input": getattr(request, "state", ""),
"fields": getattr(request, "fields", {}),
}
names = [
name
for name in source["fields"]
if receipt_totals.eligible(source, name)
]
if names:
encoded[0]["_receipt_sum_fields"] = names
return encoded
finally:
cache.clear()
cache._share_image_row_description = False
async def predict_async(self, request: PredictRequest) -> PredictOutput:
if self._in_flight >= self.max_concurrency:
raise ClairQueueFull("Clairic inference queue is full")
self._in_flight += 1
loop = asyncio.get_running_loop()
deadline = loop.time() + self.REQUEST_TIMEOUT_SECONDS
encoding: asyncio.Future[EncodedRequest] | None = None
work: list[asyncio.Task[Any]] = []
auxiliary: asyncio.Task[Any]
receipt_progress: dict[str, list[Any]] = {
"encodings": [],
"tasks": [],
"readings": [],
}
try:
self.engine.begin_request()
try:
encoding = loop.run_in_executor(
self._encoder, self._encode_request, request
)
(record, forecasts, tasks, item, eos) = await asyncio.wait_for(
asyncio.shield(encoding), timeout=max(0, deadline - loop.time())
)
except ValueError as error:
raise ClairRequestError(str(error)) from error
work = [asyncio.create_task(self._ar(record, task, eos)) for task in tasks]
if item is not None:
work.append(
asyncio.create_task(
self._typed(request, item, record.get("_media_payload"))
)
)
outputs: list[Any] = await asyncio.wait_for(
asyncio.gather(*work), timeout=max(0, deadline - loop.time())
)
predictions: dict[str, dict[str, Any]] = (
outputs[-1] if item is not None else {}
)
for name, value in outputs[: len(tasks)]:
predictions[name] = value
predictions = printed_temporal.normalize_predictions(record, predictions)
predictions = printed_numbers.normalize_predictions(record, predictions)
predictions = identifier_spelling.normalize_predictions(record, predictions)
fallback = identifier_spelling.fallback_record(record, predictions)
identifier_fallback_results = {}
if fallback is not None:
identifier_fallback_results = await asyncio.wait_for(
self._ar_all(fallback, eos), timeout=max(0, deadline - loop.time())
)
predictions = identifier_spelling.merge_fallback(
predictions, identifier_fallback_results, fallback
)
date_results = {}
date_tasks = []
if supplement := record.get("_date_supplement"):
(dated, source, names, date_eos) = supplement
names = [
name
for name in names
if isinstance(predictions.get(name, {}).get("value"), str)
and predictions[name]["value"] not in source.support
]
if supplement and names:
date_tasks = dated["_ar_tasks"]
auxiliary = asyncio.create_task(self._ar_all(dated, date_eos))
work.append(auxiliary)
try:
date_results = await asyncio.wait_for(
auxiliary,
timeout=max(
0,
min(
self.DATE_SUPPLEMENT_TIMEOUT_SECONDS,
deadline - loop.time(),
),
),
)
date_results = source.annotate_predictions(date_results, names)
predictions = source.merge_predictions(
predictions, date_results, names
)
except Exception:
(date_results, date_tasks) = ({}, [])
receipt_tasks: list[ARTask]
receipt_readings: list[dict[str, Any]]
(receipt_tasks, receipt_readings) = ([], [])
if record.get("_receipt_sum_fields"):
auxiliary = asyncio.create_task(
self._complete_receipt_sums(
request, record, predictions, eos, receipt_progress
)
)
work.append(auxiliary)
try:
(updates, receipt_tasks, receipt_readings) = await asyncio.wait_for(
auxiliary,
timeout=max(
0,
min(
self.RECEIPT_SUPPLEMENT_TIMEOUT_SECONDS,
deadline - loop.time(),
),
),
)
predictions = {**predictions, **updates}
except Exception:
pass
(receipt_tasks, receipt_readings) = (
receipt_progress["tasks"],
receipt_progress["readings"],
)
predictions = source_rows.bounded_predictions(
predictions,
record["fields"],
record.get("_list_source_bounds", {}),
zs.list_item_name,
)
merged = zs.assemble_list_predictions(record, predictions)
merged.update(image_rows.assemble(record, predictions))
merged.update(image_table_rows.assemble(record, predictions))
merged.update(image_ragged_rows.assemble(record, predictions))
merged.update(image_financial_lines.assemble(record, predictions))
annotations = record["_meta"]["extraction_contract"]["fields"]
for name, annotation in annotations.items():
if annotation.get("source_fields"):
merged[name] = table_rows.assemble(
predictions,
annotation["source_rows"],
annotation["source_fields"],
eligibility=annotation.get("source_row_eligibility", False),
)
truncated = any(
(
not predictions[n].get("valid", False)
for (n, _, _) in tasks
if not annotations.get(n, {}).get("parent")
)
)
truncated |= any(
(
not merged[name]["valid"] or not merged[name]["complete"]
for name in record.get("_financial_line_plans", {})
)
)
for name, annotation in record["_meta"]["extraction_contract"][
"fields"
].items():
if annotation.get("source_fields"):
truncated |= not merged[name]["complete"]
if (
annotation.get("image_row_fields")
or annotation.get("geometric_row_fields")
or annotation.get("ragged_row_fields")
):
truncated |= (
not merged[name]["valid"] or not merged[name]["complete"]
)
queries = annotation.get("query_fields")
if queries:
used = []
for child in queries:
used.append(predictions[child])
if predictions[child]["value"] is None:
break
valid = all((child.get("valid", True) for child in used))
complete = (
valid
and used[-1]["value"] is None
and used[-1].get("complete", True)
)
merged[name].update(valid=valid, complete=complete)
merged[name].update(
p_null=used[0]["p_null"],
p_null_method="first_item_null_probability",
p_nulls=[
child["p_null"]
for child in used
if child["value"] is not None
],
)
if all((child.get("decoder") == ar.VERSION for child in used)):
merged[name]["confidence"] = 0.0
if all(("log_probability" in child for child in used)):
log_probability = sum(
(child["log_probability"] for child in used)
)
merged[name].update(
confidence=math.exp(min(0.0, log_probability))
if complete
else 0.0,
log_probability=log_probability,
confidence_method="ar_sequence_probability_with_structural_end"
if any(
(
child.get("grounding_method")
== "source_row_bound"
for child in used
)
)
else "ar_sequence_probability",
)
merged[name]["spans"] = [
child.get("span")
for child in used
if child["value"] is not None
]
truncated |= not complete
merged = cm.merge_forecasts(merged, forecasts) if forecasts else merged
answers = {
name: {"type": spec["type"], **merged[name]}
for (name, spec) in request.fields.items()
}
state_tokens = len(
self.tokenizer.encode(request.state, add_special_tokens=False)
)
input_tokens = sum((len(ids) for (_, ids, _) in tasks)) + (
len(item.input_ids) if item is not None else 0
)
input_tokens += sum(
(v.get("retry_input_tokens", 0) for (_, v) in outputs[: len(tasks)])
)
input_tokens += sum(
(
v.get("retry_input_tokens", 0)
- v["identifier_spelling_attempt"].get("retry_input_tokens", 0)
for v in predictions.values()
if v.get("identifier_spelling_fallback")
)
)
input_tokens += sum((len(ids) for (_, ids, _) in date_tasks))
input_tokens += sum((len(ids) for (_, ids, _) in receipt_tasks))
unexecuted = record.get("_unexecuted_list_items", {})
input_tokens -= sum(unexecuted.values())
input_work = (
len(tasks)
+ (item is not None)
- len(unexecuted)
+ len(identifier_fallback_results)
+ len(date_tasks)
+ len(receipt_tasks)
)
image_id: Any = getattr(self.processor, "image_token_id", None)
image_tokens = (
tasks[0][1].count(image_id)
if record.get("_media_payload") and tasks
else list(item.input_ids).count(image_id)
if record.get("_media_payload") and item is not None
else 0
)
return PredictOutput(
answers=answers,
truncated=truncated,
state_tokens=state_tokens,
image_tokens=image_tokens,
schema_tokens=max(
0, input_tokens - (state_tokens + image_tokens) * input_work
),
input_tokens=input_tokens,
output_tokens=sum((v["tokens"] for (_, v) in outputs[: len(tasks)]))
+ sum((v["tokens"] for v in identifier_fallback_results.values()))
+ sum((v["tokens"] for v in date_results.values()))
+ sum((v["tokens"] for v in receipt_readings)),
passes=max(
[1]
+ [v["tokens"] for (_, v) in outputs[: len(tasks)]]
+ [
predictions[name]["tokens"]
for name in identifier_fallback_results
]
+ [v["tokens"] for v in date_results.values()]
+ [v["tokens"] for v in receipt_readings]
),
state_text=request.state,
)
except (ClairRequestError, ClairQueueFull):
raise
except Exception as error:
raise ClairInferenceError(
"Clairic Transformers inference failed"
) from error
finally:
await asyncio.shield(
asyncio.create_task(
self._release_request(work, encoding, receipt_progress["encodings"])
)
)
async def _release_request(
self,
work: Sequence[asyncio.Task[Any]],
encoding: asyncio.Future[Any] | None,
extra_encodings: Sequence[asyncio.Future[Any]] = (),
) -> None:
for task in work:
if not task.done():
task.cancel()
if work:
await asyncio.gather(*work, return_exceptions=True)
await self.engine.end_request()
if extra_encodings:
encoding = asyncio.gather(
*([encoding] if encoding is not None else []),
*extra_encodings,
return_exceptions=True,
)
if encoding is not None and (not encoding.done()):
encoding.add_done_callback(self._release_encoding)
else:
self._in_flight -= 1
def _release_encoding(self, future: asyncio.Future[Any]) -> None:
if not future.cancelled():
future.exception()
self._in_flight -= 1
def shutdown(self) -> None:
self._encoder.shutdown(wait=False, cancel_futures=True)
self.engine.shutdown()
@classmethod
def load(
cls,
settings: RuntimeSettings,
**model_kwargs: Any,
) -> ClairicFlashModel:
# MLX port: the tokenizer is unchanged; the processor and engine are the torch-free MLX versions.
from transformers import AutoTokenizer
from .mlx_engine import MLXEngine
from .processor import ClairicProcessor
path = Path(settings.model_path)
tokenizer = AutoTokenizer.from_pretrained(path, local_files_only=True)
validate_eos(path, tokenizer)
for token in ("null", "false"):
if tokenizer.encode(token, add_special_tokens=False) != [
tokenizer.convert_tokens_to_ids(token)
]:
raise ValueError(f"{token} must be one token for confidence scoring")
processor = ClairicProcessor(path, tokenizer, 1600000)
engine = MLXEngine(path, tokenizer, **model_kwargs)
return cls(
engine,
tokenizer,
processor,
settings,
json.loads((path / "clair_head_config.json").read_text()),
)
# Modified from the Cloudflare Clairic release (Apache-2.0) for the MLX port:
# - `media_payload` keeps pixel values as NumPy arrays (rounded through bfloat16, as upstream does) and adds
# a content key for the engine's per-request image-feature cache;
# - `load` builds the MLX engine (`mlx_engine.py`) and the torch-free processor (`processor.py`) instead of
# the PyTorch engine and the Transformers processor, and takes engine options instead of device / dtype.
# Everything else is unchanged.
from __future__ import annotations
from collections.abc import Callable, Collection, Sequence
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .api import ClairImage
from .documents import ClairDocument
from PIL import Image
import asyncio
import concurrent.futures
import json
import logging
import math
import uuid
from decimal import Decimal
from pathlib import Path
from typing import Any, cast
from .api import MAX_IMAGES, PredictOutput, PredictRequest
from .errors import ClairInferenceError, ClairQueueFull, ClairRequestError
from .settings import RuntimeSettings
from . import (
MAX_OUTPUT_TOKENS,
ar,
batch_completion,
contracts,
entity_scope,
financial_scope,
grounding,
identifier_scope,
identifier_spelling,
image_financial_lines,
image_focus,
image_identifier,
image_party_names,
image_ragged_rows,
image_rows,
image_table_rows,
ordinary_list_completion,
pdf_number_focus,
printed_numbers,
printed_temporal,
receipt_totals,
row_omission,
source_dates,
source_rows,
table_rows,
)
from . import lists as extraction
from . import model as cm
from . import schema as zs
from .media_cache import ImageCache
from .media import batch_image_data, media_warmup_groups
if TYPE_CHECKING:
MediaPayload = list[dict[str, Any]]
ARTask = tuple[str, list[int], MediaPayload | None]
EncodedRequest = tuple[
dict[str, Any], dict[str, list[str]], list[ARTask], cm.EncodedRecord | None, int
]
logger = logging.getLogger(__name__)
def common_prefix(tasks: Sequence[ARTask]) -> list[int]:
(first, last) = (min((t[1] for t in tasks)), max((t[1] for t in tasks)))
length = next(
(i for (i, (a, b)) in enumerate(zip(first, last)) if a != b),
min(len(first), len(last)),
)
return first[:length]
def expand_request(
request: PredictRequest,
*,
date_role_guidance: bool = True,
source_row_eligibility: bool = False,
local_scoped_pairs: bool = False,
fee_guidance: bool = False,
service_guidance: bool = False,
literal_payment_rows: bool = False,
) -> tuple[dict[str, Any], dict[str, list[str]]]:
contracts.validate_public_fields(request.fields)
(fields, forecasts) = zs.expand_forecasts(
financial_scope.prepare(
request.fields,
has_media=bool(request.images or request.documents),
fee_guidance=fee_guidance,
service_guidance=service_guidance,
)
)
zs.validate_fields(fields)
zs.validate_reference_date(request.reference_date)
fields = {n: dict(s) for (n, s) in fields.items()}
for spec in fields.values():