Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic
- SGLang
How to use Cloudflare/clairic with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic
Download clairic_runtime/model.py from Cloudflare/clairic: direct link, hf CLI and curl.
- Browser
- Download file 110 kB
-
https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/model.py
- Command line
-
hf download hf://Cloudflare/clairic/clairic_runtime/model.py
-
curl -L -o model.py https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/model.py
110 kB
from __future__ import annotations
import calendar
import datetime
import functools
import math
import os
import re
from bisect import bisect_left, bisect_right
from dataclasses import dataclass, field
from decimal import ROUND_HALF_UP, Decimal
from typing import Any, cast
import torch
from torch import nn
from torch.nn import functional
from . import schema as zs
SYSTEM_PREFIX = "<|im_start|>system\n" + zs.SYSTEM_PREAMBLE + "\nFields:\n"
EVIDENCE_SYSTEM_PREFIX = (
"<|im_start|>system\n" + zs.EVIDENCE_SYSTEM_PREAMBLE + "\nFields:\n"
)
USER_PREFIX = "<|im_end|>\n<|im_start|>user\n"
ASSISTANT_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n"
ANSWER_CUE = "\nAnswer:"
@dataclass(frozen=True)
class PromptLayout:
schema_header: str
query_header: str
answer_cue: str
slot_header: str
PROMPT_LAYOUTS = (
PromptLayout("", "", ANSWER_CUE, ""),
PromptLayout(
"Field definition:\n", "\nField query:", "\nValue:", "\nAnswer field:"
),
PromptLayout(
"Requested field:\n", "\nRequested field:", "\nResult:", "\nResult for field:"
),
)
PROMPT_VARIANTS = (0, 1, 2)
def prompt_layout(
template_variant: int = 0, complete_field: bool = True
) -> PromptLayout:
if (
not isinstance(template_variant, int)
or isinstance(template_variant, bool)
or template_variant not in PROMPT_VARIANTS
):
raise ValueError("template_variant must be 0, 1, or 2")
if template_variant and (not complete_field):
raise ValueError("alternative layouts require the evidence construction")
return PROMPT_LAYOUTS[template_variant]
QUERY_TOKENS = 256
MAX_SPAN = 64
TRIM_CLASSES = 8
TRIM_INIT = 2.0
MIN_INPUT_TOKENS = 16
BINS = 256
POINTER_WIDTH = 256
NUMBER_DIGITS = 12
(MIN_EXPONENT, MAX_EXPONENT) = (-12, 15)
EXPONENT_CLASSES = MAX_EXPONENT - MIN_EXPONENT + 2
YEAR_DIGITS = 4
(YEAR_LOW, YEAR_HIGH) = (1800, 2199)
YEAR_OUTSIDE = YEAR_HIGH - YEAR_LOW + 1
YEAR_CLASSES = YEAR_OUTSIDE + 1
LIST_SLOTS = zs.MAX_LIST_ITEMS
ITEM_CUES = tuple((f"\n{index}." for index in range(1, LIST_SLOTS + 1)))
(INSIDE, BELOW, ABOVE) = (0, 1, 2)
OVERFLOW_INIT = -5.0
QUERY_ORDER_RATE = 0.8
LOOP_RANK = 256
LEXICAL_DIGITS = 4
ESTIMATE_GATE_INIT = 0.0
ESTIMATE_SPREAD_LIMIT = 2.0
(EDGE_NEGATIVE, EDGE_ZERO, EDGE_POSITIVE) = (0, 1, 2)
(TAG_OUTSIDE, TAG_BEGIN, TAG_INSIDE) = (0, 1, 2)
NUMBER_WIDTH = 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10
GROUP_DIGITS = 3
GROUP_CLASSES = 10**GROUP_DIGITS
(GROUP_LOW, GROUP_HIGH) = (MIN_EXPONENT // GROUP_DIGITS, MAX_EXPONENT // GROUP_DIGITS)
NUMBER_GROUPS = GROUP_HIGH - GROUP_LOW + 1
GROUP_RANK = 256
COPY_CATEGORIES: tuple[str | None, ...] | None = ("extraction", None)
COPY_GATE_INIT = -3.0
NUMERAL_RUN = re.compile(
"(?:(?<![\\d.])\\.(?=\\d))?\\d+(?:[,.'\\u00a0\\u202f\\u2009\\u2007 \\u066b\\u066c]\\d+)*"
)
NUMERAL_SEPARATORS = ",.'\xa0\u202f\u2009\u2007 ٫٬"
NUMERAL_READINGS = (
(re.compile("\\d{1,3}(?:,\\d{3})+(?:\\.\\d+)?|\\d+(?:\\.\\d+)?|\\.\\d+"), ",", "."),
(
re.compile(
"\\d{1,3}(?:[.\\u00a0\\u202f\\u2009\\u2007' ]\\d{3})+(?:,\\d+)?|\\d+(?:,\\d+)?"
),
".\xa0\u202f\u2009\u2007' ",
",",
),
(
re.compile("\\d{1,3}(?:[\\u00a0\\u202f\\u2009\\u2007' ]\\d{3})+(?:\\.\\d+)?"),
"\xa0\u202f\u2009\u2007' ",
".",
),
(re.compile("\\d{1,2}(?:,\\d{2})+,\\d{3}(?:\\.\\d+)?"), ",", "."),
(
re.compile("\\d{1,3}(?:\\u066c\\d{3})+(?:\\u066b\\d+)?|\\d+\\u066b\\d+"),
"٬",
"٫",
),
(re.compile("\\d{1,2}(?:\\u066c\\d{2})+\\u066c\\d{3}(?:\\u066b\\d+)?"), "٬", "٫"),
)
NUMERAL_SCALES = {
"thousand": 3,
"thousands": 3,
"k": 3,
"K": 3,
"million": 6,
"millions": 6,
"mn": 6,
"mln": 6,
"M": 6,
"MM": 6,
"billion": 9,
"billions": 9,
"bn": 9,
"B": 9,
"trillion": 12,
"tn": 12,
"lakh": 5,
"lakhs": 5,
"lac": 5,
"lacs": 5,
"crore": 7,
"crores": 7,
"cr": 7,
"Cr": 7,
"Mn": 6,
"Bn": 9,
"Lakh": 5,
"Lakhs": 5,
"Crore": 7,
"Crores": 7,
"Million": 6,
"Billion": 9,
"Thousand": 3,
"Trillion": 12,
"MILLION": 6,
"BILLION": 9,
"ألف": 3,
"مليون": 6,
"مليار": 9,
"हज़ार": 3,
"लाख": 5,
"करोड़": 7,
}
NUMERAL_SCALE = re.compile(
"(?:[ \\u00a0\\u202f\\u2009]?("
+ "|".join(
sorted(
(
word
for word in NUMERAL_SCALES
if len(word) > 2
or word in ("mn", "bn", "tn", "cr", "Cr", "Mn", "Bn", "MM")
),
key=len,
reverse=True,
)
)
+ ")|(k|K|M|B))(?![^\\W\\d_])"
)
NUMERAL_SPAN = 48
CONVENTION_LANGUAGE = re.compile(
"\\b(?:Rechnung|Gesamtbetrag|Betrag|Lieferschein|Factuur|Bedrag|Totaal|Pakbon|Fattura|Importo|Totale|Fatura|Nota fiscal|Factura|Importe|Albar\\u00e1n|Facture|Montant)\\b"
)
CONVENTION_LANGUAGE_US = re.compile("\\b(?:M\\u00e9xico|Mexico|MXN|CFDI|RFC)\\b")
CONVENTION_STATED = {
"eu": re.compile(
"european (?:number )?(?:notation|format)|(?:dot|period|point)s? (?:separates?|for|marks?) (?:the )?thousands|comma (?:separates?|for|marks?|as) (?:the )?decimal|comma decimals?|decimal comma|dot thousands|1\\.234,56|1 234,56",
re.IGNORECASE,
),
"us": re.compile(
"comma (?:separates?|for|marks?) (?:the )?thousands|(?:dot|period|point) (?:separates?|for|marks?|as) (?:the )?decimal|comma thousands|dot decimals?|1,234\\.56",
re.IGNORECASE,
),
}
COPY_CONFIDENT = 0.9
COPY_MIN_CONFIDENCE = 0.5
SLOT_TEXT = "_"
SLOT_COUNTS = {"text": 80, "text_list": 64, "number": 32}
SLOT_CUE_TOKENS = 32
DATE_WIDTH = YEAR_DIGITS * 10 + YEAR_CLASSES + 12 + 31
TIME_WIDTH = 24 + 60 + 60
MASKED = -1000000000.0
SCALAR_KINDS = ("text", "number", "date", "time", "datetime")
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
MEDIA_BATCH_KEYS = ("pixel_values", "image_grid_thw")
MEDIA_TOKEN_KEYS = ("mm_token_type_ids",)
LIST_KINDS = ("number_list", "date_list")
def head_kind(spec: dict[str, Any]) -> str:
kind = spec["type"]
if kind == "duration":
return "number"
if kind == "list":
return f"{spec['items']}_list"
return kind
def number_components(value: float) -> tuple[int, int, list[int]] | None:
value = float(value)
if not math.isfinite(value):
return None
if value == 0:
return (0, 0, [0] * NUMBER_DIGITS)
(mantissa, exponent) = f"{abs(value):.{NUMBER_DIGITS - 1}e}".split("e")
power = int(exponent)
if not MIN_EXPONENT <= power <= MAX_EXPONENT:
return None
return (
int(value < 0),
power - MIN_EXPONENT + 1,
[int(digit) for digit in mantissa.replace(".", "")],
)
def edge_number(value: float) -> tuple[int, int, list[int], float]:
if value == 0 or not math.isfinite(value):
return (EDGE_ZERO, 1, [0] * LEXICAL_DIGITS, 0.0)
(mantissa, exponent) = f"{abs(value):.{LEXICAL_DIGITS + 8}e}".split("e")
(power, digits) = (int(exponent), mantissa.replace(".", ""))
sign = EDGE_NEGATIVE if value < 0 else EDGE_POSITIVE
if power < MIN_EXPONENT:
return (sign, 1, [0] * LEXICAL_DIGITS, 0.0)
if power > MAX_EXPONENT:
return (sign, EXPONENT_CLASSES - 1, [9] * LEXICAL_DIGITS, 1.0)
return (
sign,
power - MIN_EXPONENT + 1,
[int(digit) for digit in digits[:LEXICAL_DIGITS]],
float("0." + digits[LEXICAL_DIGITS:]),
)
@functools.lru_cache(maxsize=8192)
def _edge_parts(
low: float, high: float, scale: str, bins: int
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]]:
parts = [
edge_number(zs.from_unit(index / bins, low, high, scale))
for index in range(bins + 1)
]
return (
tuple((part[0] for part in parts)),
tuple((part[1] for part in parts)),
tuple((digit for part in parts for digit in part[2])),
tuple((part[3] for part in parts)),
)
def lexical_estimate(spec: dict[str, Any]) -> bool:
return not zs.is_date_estimate(spec) or zs.date_granularity(spec) == "year"
def estimate_edge_parts(
spec: dict[str, Any], bins: int = BINS
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[float, ...]] | None:
if not lexical_estimate(spec):
return None
(low, high, scale) = zs.estimate_range(spec)
return _edge_parts(float(low), float(high), scale, bins)
@dataclass
class EncodedField:
name: str
kind: str
query: int
items: list[int] = field(default_factory=list)
prefix: tuple[int, int, tuple[int, ...]] | None = None
window: tuple[int, int] | None = None
answer_slots: list[int] = field(default_factory=list)
@dataclass(frozen=True)
class Commit:
text: str
done: bool = False
prefix: tuple[int, int, tuple[int, ...]] | None = None
probability: float = 1.0
@dataclass
class EncodedRecord:
input_ids: list[int]
input_start: int
offsets: list[tuple[int, int]]
fields: list[EncodedField]
text: str
record: dict[str, Any]
media: dict[str, Any] | None = None
image_expansion: int = 0
@dataclass
class EncodedMedia:
ids: list[int]
media: dict[str, Any] | None = None
expansion: int = 0
@property
def length(self) -> int:
return len(self.ids) + self.expansion
def configure_image_processor(processor: Any, max_pixels: int) -> dict[str, Any]:
image_processor = getattr(processor, "image_processor", None)
if image_processor is None or max_pixels <= 0:
raise ValueError("an image processor and positive pixel budget are required")
legacy = hasattr(image_processor, "max_pixels")
size = getattr(image_processor, "size", {})
sized = "longest_edge" in size
if not legacy and (not sized):
raise ValueError(
f"unsupported image pixel configuration: {type(image_processor).__name__}"
)
if legacy:
image_processor.max_pixels = max_pixels
if sized:
size["longest_edge"] = max_pixels
actual = size["longest_edge"] if sized else image_processor.max_pixels
if actual != max_pixels or (legacy and image_processor.max_pixels != max_pixels):
raise ValueError(
"the image processor did not retain its requested pixel budget"
)
return {"class": type(image_processor).__name__, "max_pixels": actual}
def encode_media(
tokenizer: Any,
images: list[Any],
processor: Any = None,
image_token_counts: list[int] | None = None,
) -> EncodedMedia:
if not images:
return EncodedMedia([])
text = IMAGE_PLACEHOLDER * len(images) + "\n"
if image_token_counts is not None:
if len(image_token_counts) != len(images) or any(
(count < 1 for count in image_token_counts)
):
raise ValueError("every image needs a positive token count")
return EncodedMedia(
_tokens(tokenizer, text),
None,
sum((count - 1 for count in image_token_counts)),
)
if processor is None:
raise ValueError("records with images need the model processor")
encoded = processor(text=[text], images=list(images), return_tensors="pt")
media = {key: encoded[key] for key in MEDIA_BATCH_KEYS if key in encoded}
for key in MEDIA_TOKEN_KEYS:
if key in encoded:
media[key] = encoded[key][0].tolist()
return EncodedMedia(encoded["input_ids"][0].tolist(), media)
def _tokens(tokenizer: Any, text: str) -> list[int]:
return tokenizer(text, add_special_tokens=False).input_ids
def _input_tokens(tokenizer: Any, text: str) -> tuple[list[int], list[tuple[int, int]]]:
try:
encoding = tokenizer(
text,
add_special_tokens=False,
return_offsets_mapping=True,
split_special_tokens=True,
)
except TypeError:
encoding = tokenizer(
text, add_special_tokens=False, return_offsets_mapping=True
)
return (list(encoding.input_ids), [tuple(pair) for pair in encoding.offset_mapping])
def schema_line(name: str, spec: dict[str, Any], complete_field: bool = False) -> str:
return "- " + zs.field_line(name, spec, preserve_instructions=complete_field) + "\n"
def reference_line(record: dict[str, Any]) -> str:
reference = record.get("reference_date")
return f"Reference date: {reference}\n" if reference else ""
def query_block(
tokenizer: Any,
name: str,
spec: dict[str, Any],
commit: Commit | None = None,
complete_field: bool = False,
template_variant: int = 0,
) -> tuple[list[int], int, list[int]]:
layout = prompt_layout(template_variant, complete_field)
line = _tokens(
tokenizer,
"\n" + zs.field_line(name, spec, preserve_instructions=complete_field),
)
ids = (
_tokens(tokenizer, layout.query_header)
+ (line if complete_field else line[:QUERY_TOKENS])
+ _tokens(tokenizer, layout.answer_cue)
)
if commit is not None:
ids.extend(_tokens(tokenizer, " " + commit.text))
query = len(ids) - 1
items: list[int] = []
if head_kind(spec) in LIST_KINDS and commit is None:
for cue in ITEM_CUES:
ids.extend(_tokens(tokenizer, cue))
items.append(len(ids) - 1)
return (ids, query, items)
def clean_cut(
text: str, offsets: list[tuple[int, int]], budget: int, lookback: int = 32
) -> int:
lowest = max(0, budget - lookback)
for keep in range(budget, lowest, -1):
end = offsets[keep - 1][1]
if end >= len(text) or (
end > 0 and text[end].isspace() and (not text[end - 1].isspace())
):
return keep
for keep in range(budget, lowest, -1):
end = offsets[keep - 1][1]
if end > 0 and (text[end].isspace() or text[end - 1].isspace()):
return keep
return budget
def char_span_to_tokens(
starts: list[int], ends: list[int], char_start: int, char_end: int
) -> tuple[int, int] | None:
first = bisect_right(ends, char_start)
last = bisect_left(starts, char_end) - 1
if first >= len(starts) or last < first:
return None
return (first, last)
def numeral_values(printed: str, convention: str | None = None) -> list[Decimal]:
values: list[Decimal] = []
both = convention is not None and all(
(form.fullmatch(printed) for (form, _, _) in NUMERAL_READINGS[:2])
)
for index, (form, separators, mark) in enumerate(NUMERAL_READINGS):
if both and index == (1 if convention == "us" else 0):
continue
if form.fullmatch(printed):
plain = printed.translate(
{ord(character): None for character in separators}
).replace(mark, ".")
try:
value = Decimal(plain)
except ArithmeticError:
continue
if value not in values:
values.append(value)
return values
@functools.lru_cache(maxsize=256)
def numeral_convention(text: str) -> str | None:
stated = {
name
for (name, pattern) in CONVENTION_STATED.items()
if pattern.search(text or "")
}
if len(stated) == 1:
return stated.pop()
shown: list[str] = []
for run in NUMERAL_RUN.finditer(text or ""):
printed = run.group()
if not any((mark in printed for mark in ",.")):
continue
(us, eu) = (
bool(form.fullmatch(printed)) for (form, _, _) in NUMERAL_READINGS[:2]
)
if us != eu:
shown.append("us" if us else "eu")
if len(shown) >= 2 and len(set(shown)) == 1:
return shown[0]
if (
not shown
and CONVENTION_LANGUAGE.search(text or "")
and (not CONVENTION_LANGUAGE_US.search(text or ""))
):
return "eu"
return None
@functools.lru_cache(maxsize=256)
def _numeral_candidates(text: str) -> tuple[tuple[list[Decimal], int, int, bool], ...]:
found: list[tuple[list[Decimal], int, int, bool]] = []
for run in NUMERAL_RUN.finditer(text or ""):
(begin, finish) = (run.start(), run.end())
cuts = [
index
for index in range(begin, finish)
if text[index] in NUMERAL_SEPARATORS and index > begin
]
starts = [begin] + [index + 1 for index in cuts]
ends = cuts + [finish]
convention = numeral_convention(text)
scale = NUMERAL_SCALE.match(text, finish)
exponent = (
None if scale is None else NUMERAL_SCALES[scale.group(1) or scale.group(2)]
)
run_found = []
for start in starts:
for end in ends[bisect_left(ends, start + 1) :]:
if end - start > NUMERAL_SPAN:
break
values = numeral_values(text[start:end], convention)
if not values:
continue
if end == finish and exponent is not None:
scaled = [value.scaleb(exponent) for value in values]
run_found.append(
(
scaled + [value for value in values if value not in scaled],
start,
cast(re.Match[str], scale).end(),
)
)
else:
run_found.append((values, start, end))
by_start: dict[int, tuple[list[Decimal], int, int]] = {}
for item in run_found:
if item[1] not in by_start or item[2] > by_start[item[1]][2]:
by_start[item[1]] = item
(position, whole) = (begin, set())
while True:
longest = by_start.get(position)
following = bisect_left(
starts, position + 1 if longest is None else longest[2]
)
if longest is not None:
whole.add((longest[1], longest[2]))
if following >= len(starts):
break
position = starts[following]
found.extend(
(
(values, start, end, (start, end) in whole)
for (values, start, end) in run_found
)
)
return tuple(found)
def read_numerals(
text: str, every: bool = False
) -> list[tuple[list[Decimal], int, int]]:
found = [
(values, start, end)
for (values, start, end, whole) in _numeral_candidates(text)
if every or whole
]
return sorted(found, key=lambda item: (item[1], -item[2]))
def numeral_spans(text: str, value: Any, every: bool = True) -> list[tuple[int, int]]:
try:
target = abs(Decimal(str(value)))
except ArithmeticError:
return []
candidates = _numeral_candidates(text)
whole = [
(start, end)
for (values, start, end, longest) in candidates
if longest and target in values
]
if whole or not every:
return sorted(whole)
return sorted(
{(start, end) for (values, start, end, _) in candidates if target in values},
key=lambda item: (item[0], -item[1]),
)
def slot_token_id(tokenizer: Any) -> int:
ids = _tokens(tokenizer, SLOT_TEXT)
if len(ids) != 1:
raise ValueError(f"answer slots need {SLOT_TEXT!r} to be one token")
return ids[0]
def slot_block(
tokenizer: Any,
name: str,
spec: dict[str, Any],
complete_field: bool = False,
template_variant: int = 0,
) -> list[int]:
count = SLOT_COUNTS.get(head_kind(spec))
layout = prompt_layout(template_variant, complete_field)
if count is None:
return []
cue = _tokens(tokenizer, "\n" + name + ":")
return (
_tokens(tokenizer, layout.slot_header)
+ (cue if complete_field else cue[:SLOT_CUE_TOKENS])
+ [slot_token_id(tokenizer)] * count
)
def system_prefix(record: dict[str, Any], text_answer_slots: bool = False) -> str:
return (
EVIDENCE_SYSTEM_PREFIX
if evidence_construction(record, text_answer_slots)
else SYSTEM_PREFIX
)
def evidence_construction(
record: dict[str, Any], text_answer_slots: bool = False
) -> bool:
return (
text_answer_slots
or (record.get("_meta") or {}).get("construction") == zs.EVIDENCE_CONSTRUCTION
)
def encode_record(
tokenizer: Any,
record: dict[str, Any],
max_length: int = 8192,
media: EncodedMedia | None = None,
commits: dict[str, Commit] | None = None,
answer_slots: bool = False,
text_answer_slots: bool = False,
template_variant: int = 0,
) -> EncodedRecord | None:
if text_answer_slots and (not answer_slots):
raise ValueError("text_answer_slots needs answer_slots")
commits = commits or {}
if record.get("images") and media is None:
raise ValueError("records with images need their encoded media")
media = media or EncodedMedia([])
items = list(record["fields"].items())
complete_field = evidence_construction(record, text_answer_slots)
layout = prompt_layout(template_variant, complete_field)
ids = _tokens(tokenizer, system_prefix(record, text_answer_slots))
for name, spec in items:
ids.extend(_tokens(tokenizer, layout.schema_header))
ids.extend(_tokens(tokenizer, schema_line(name, spec, complete_field)))
ids.extend(_tokens(tokenizer, reference_line(record)))
ids.extend(_tokens(tokenizer, USER_PREFIX))
media_start = len(ids)
ids.extend(media.ids)
input_start = len(ids)
suffix = _tokens(tokenizer, ASSISTANT_SUFFIX)
blocks = [
query_block(
tokenizer, name, spec, commits.get(name), complete_field, template_variant
)
for (name, spec) in items
]
slot_blocks: dict[int, list[int]] = {}
if answer_slots and (media.ids or text_answer_slots):
for index, (name, spec) in enumerate(items):
if not (name in commits and commits[name].done):
block = slot_block(
tokenizer, name, spec, complete_field, template_variant
)
if block:
slot_blocks[index] = block
budget = (
max_length
- len(ids)
- media.expansion
- len(suffix)
- sum((len(block[0]) for block in blocks))
- sum((len(block) for block in slot_blocks.values()))
)
if budget < MIN_INPUT_TOKENS:
return None
text = record["input"]
if media.ids and (not text.strip()):
text = ""
(input_ids, offsets) = (_tokens(tokenizer, "\n"), [(0, 0)])
else:
(input_ids, offsets) = _input_tokens(tokenizer, text)
if len(input_ids) > budget:
keep = clean_cut(text, offsets, budget)
text = text[: offsets[keep - 1][1]]
(input_ids, offsets) = _input_tokens(tokenizer, text)
if len(input_ids) > budget:
(input_ids, offsets) = (input_ids[:budget], offsets[:budget])
text = text[: offsets[-1][1]]
if not input_ids:
return None
ids.extend(input_ids)
ids.extend(suffix)
order = list(range(len(items)))
done = {
index
for (index, (name, _)) in enumerate(items)
if name in commits and commits[name].done
}
if done:
order = [index for index in order if index in done] + [
index for index in order if index not in done
]
queries: dict[int, int] = {}
slots: dict[int, list[int]] = {}
for index in order:
(block, query, item_offsets) = blocks[index]
start = len(ids)
ids.extend(block)
queries[index] = start + query
slots[index] = [start + offset for offset in item_offsets]
answer_positions: dict[int, list[int]] = {}
for index in order:
if index in slot_blocks:
ids.extend(slot_blocks[index])
count = SLOT_COUNTS[head_kind(items[index][1])]
answer_positions[index] = list(range(len(ids) - count, len(ids)))
fields: list[EncodedField] = []
for index, (name, spec) in enumerate(items):
if index in done:
continue
encoded = EncodedField(name, head_kind(spec), queries[index], slots[index])
if index in answer_positions:
encoded.answer_slots = answer_positions[index]
commit = commits.get(name)
if (
commit is not None
and commit.prefix is not None
and (encoded.kind in ("number", "estimate"))
):
encoded.prefix = commit.prefix
if encoded.kind == "estimate":
encoded.window = prefix_window(spec, commit.prefix)
fields.append(encoded)
if not fields:
return None
record_media = None
if media.media is not None:
record_media = {**media.media, "token_offset": media_start}
return EncodedRecord(
ids, input_start, offsets, fields, text, record, record_media, media.expansion
)
def span_base(text: str, starts: list[int], ends: list[int], first: int) -> int:
base = starts[first]
while base < ends[first] - 1 and text[base].isspace():
base += 1
return base
def trimmed_span(
text: str,
offsets: list[tuple[int, int]],
first: int,
last: int,
trims: list[list[int]] | None,
) -> tuple[int, int]:
(char_start, char_end) = (offsets[first][0], offsets[last][1])
while char_start < offsets[first][1] - 1 and text[char_start].isspace():
char_start += 1
if trims is not None:
start = max(
char_start, min(char_start + int(trims[first][0]), offsets[first][1] - 1)
)
end = min(char_end, max(char_end - int(trims[last][1]), offsets[last][0] + 1))
if start < end:
(char_start, char_end) = (start, end)
while char_start < char_end - 1 and text[char_start].isspace():
char_start += 1
return (char_start, char_end)
def plain_number(value: Any) -> str:
number = Decimal(str(value))
if not number.is_finite():
raise ValueError("A plain number must be finite.")
text = format(number, "f")
return text.rstrip("0").rstrip(".") if "." in text else text
def prefix_window(
spec: dict[str, Any], prefix: tuple[int, int, tuple[int, ...]], bins: int = BINS
) -> tuple[int, int] | None:
(sign, exponent_class, digits) = prefix
power = exponent_class - 1 + MIN_EXPONENT
unit = Decimal(1).scaleb(power - len(digits) + 1)
start = Decimal(int("".join((str(digit) for digit in digits)))) * unit
(first, last) = (float(start), float(start + unit))
if sign:
(first, last) = (-last, -first)
(low, high, scale) = zs.estimate_range(spec)
(lower, upper) = sorted(
(zs.raw_unit(first, low, high, scale), zs.raw_unit(last, low, high, scale))
)
if upper <= 0.0 or lower >= 1.0:
return None
first_bin = max(0, min(bins - 1, math.floor(lower * bins)))
last_bin = max(first_bin, min(bins - 1, math.ceil(upper * bins) - 1))
return (first_bin, last_bin)
def span_scores(
start: torch.Tensor, end: torch.Tensor, length_bias: torch.Tensor
) -> torch.Tensor:
width = length_bias.shape[0]
padded = functional.pad(end, (0, width - 1), value=float("-inf"))
return start.unsqueeze(-1) + padded.unfold(-1, width, 1) + length_bias
DIGIT_SLOTS = NUMBER_DIGITS + YEAR_DIGITS + 2 + 2 + 2 + 2
(YEAR_SLOT, DAY_SLOT, HOUR_SLOT, MINUTE_SLOT, SECOND_SLOT) = (
NUMBER_DIGITS,
NUMBER_DIGITS + YEAR_DIGITS,
NUMBER_DIGITS + YEAR_DIGITS + 2,
NUMBER_DIGITS + YEAR_DIGITS + 4,
NUMBER_DIGITS + YEAR_DIGITS + 6,
)
MONTH_WORDS = tuple((" " + name for name in calendar.month_name[1:]))
SIGN_WORDS = (" ", " -")
ROUTE_KINDS = (
"text",
"tags",
"number",
"date",
"time",
"estimate",
"number_list",
"date_list",
)
def two_digit_index(values: range) -> torch.Tensor:
return torch.tensor(
[[value // 10 for value in values], [value % 10 for value in values]],
dtype=torch.long,
)
def compose_two_digits(slots: torch.Tensor, index: torch.Tensor) -> torch.Tensor:
return slots[..., 0, :].index_select(-1, index[0]) + slots[..., 1, :].index_select(
-1, index[1]
)
def compose_years(slots: torch.Tensor, index: torch.Tensor) -> torch.Tensor:
inside = cast(
torch.Tensor,
sum(
(
slots[..., position, :].index_select(-1, index[position])
for position in range(YEAR_DIGITS)
)
),
)
total = torch.logsumexp(slots, dim=-1).sum(dim=-1)
gap = (torch.logsumexp(inside, dim=-1) - total).clamp(max=-1e-06)
outside = total + torch.log(-torch.expm1(gap))
return torch.cat([inside, outside.unsqueeze(-1)], dim=-1)
class ClairicHead(nn.Module):
digit_rows: torch.Tensor
month_rows: torch.Tensor
sign_rows: torch.Tensor
year_index: torch.Tensor
day_index: torch.Tensor
hour_index: torch.Tensor
sixty_index: torch.Tensor
def __init__(
self,
hidden_size: int,
bins: int = BINS,
pointer_width: int = POINTER_WIDTH,
readout: str = "linear",
route_width: int = 1024,
route_layers: int = 2,
route_heads: int = 8,
slot_rank: int = 32,
estimate_lexical: bool = False,
span_trim: bool = False,
loop_feedback: bool = False,
loop_scale: float = 1.0,
number_groups: bool = False,
number_pointer: bool = False,
answer_slots: bool = False,
answer_slot_routing: bool = False,
) -> None:
super().__init__()
if readout not in ("linear", "lexical"):
raise ValueError(f"unknown readout {readout!r}")
if (number_groups or number_pointer) and readout != "lexical":
raise ValueError(
"number_groups and number_pointer need the lexical readout"
)
if estimate_lexical and readout != "lexical":
raise ValueError("estimate_lexical needs the lexical readout")
if answer_slot_routing and (readout != "lexical" or not answer_slots):
raise ValueError(
"answer_slot_routing needs lexical readout and answer_slots"
)
self.config: dict[str, Any] = {
"hidden_size": hidden_size,
"bins": bins,
"pointer_width": pointer_width,
}
if readout == "lexical":
self.config.update(
readout=readout,
route_width=route_width,
route_layers=route_layers,
route_heads=route_heads,
slot_rank=slot_rank,
)
if estimate_lexical:
self.config.update(estimate_lexical=True)
if span_trim:
self.config.update(span_trim=True)
if loop_feedback:
self.config.update(loop_feedback=True, loop_scale=float(loop_scale))
self.number_groups = False
self.number_pointer = False
self.answer_slots = bool(answer_slots)
if answer_slots:
self.config["answer_slots"] = True
self.answer_slot_routing = bool(answer_slot_routing)
if answer_slot_routing:
self.config["answer_slot_routing"] = True
self.slot_route_scale = nn.Parameter(torch.ones(len(ROUTE_KINDS)))
self.readout = readout
self.loop_feedback = loop_feedback
self.loop_scale = float(loop_scale)
self.span_trim = span_trim
(self.estimate_lexical,) = (estimate_lexical,)
(self.pointer_width,) = (pointer_width,)
self.norm = nn.LayerNorm(hidden_size)
self.token_norm = nn.LayerNorm(hidden_size)
self.null = nn.Linear(hidden_size, 1)
self.span_query = nn.Linear(hidden_size, 2 * pointer_width)
self.span_key = nn.Linear(hidden_size, 2 * pointer_width)
self.length_bias = nn.Parameter(
-0.05 * torch.arange(MAX_SPAN, dtype=torch.float32)
)
self.tag_query = nn.Linear(hidden_size, 2 * pointer_width)
self.tag_key = nn.Linear(hidden_size, pointer_width)
self.tag_bias = nn.Parameter(torch.tensor([-2.0, -2.0]))
if span_trim:
self.trim = nn.Linear(hidden_size, 2 * TRIM_CLASSES)
nn.init.zeros_(self.trim.weight)
with torch.no_grad():
self.trim.bias.copy_(
torch.tensor(([TRIM_INIT] + [0.0] * (TRIM_CLASSES - 1)) * 2)
)
if readout == "linear":
self.number = nn.Linear(hidden_size, NUMBER_WIDTH)
self.date = nn.Linear(hidden_size, DATE_WIDTH)
self.time = nn.Linear(hidden_size, TIME_WIDTH)
else:
self.route_in = nn.Linear(hidden_size, route_width, bias=False)
self.kind_embedding = nn.Embedding(
len(ROUTE_KINDS) + len(LIST_KINDS), route_width
)
self.memory_in = nn.Linear(hidden_size, route_width, bias=False)
self.route_layers = nn.ModuleList(
[
nn.TransformerDecoderLayer(
d_model=route_width,
nhead=route_heads,
dim_feedforward=4 * route_width,
dropout=0.0,
activation="gelu",
batch_first=True,
norm_first=True,
)
for _ in range(route_layers)
]
)
self.route_norm = nn.LayerNorm(route_width)
self.route_out = nn.Linear(route_width, hidden_size)
nn.init.zeros_(self.route_out.weight)
nn.init.zeros_(self.route_out.bias)
self.route_gate = nn.Parameter(torch.tensor(-2.0))
self.register_buffer(
"digit_rows", torch.randn(10, hidden_size) / math.sqrt(hidden_size)
)
self.register_buffer(
"month_rows", torch.randn(12, hidden_size) / math.sqrt(hidden_size)
)
self.register_buffer(
"sign_rows", torch.randn(2, hidden_size) / math.sqrt(hidden_size)
)
self.lexical_scale = nn.Parameter(torch.full((3,), math.log(10.0)))
self.slot_down = nn.Linear(
hidden_size, (DIGIT_SLOTS + 2) * slot_rank, bias=False
)
self.slot_up = nn.Parameter(torch.zeros(DIGIT_SLOTS, slot_rank, 10))
self.month_up = nn.Parameter(torch.zeros(slot_rank, 12))
self.sign_up = nn.Parameter(torch.zeros(slot_rank, 2))
self.slot_residual = nn.Linear(route_width, DIGIT_SLOTS * 10 + 12 + 2)
nn.init.zeros_(self.slot_residual.weight)
nn.init.zeros_(self.slot_residual.bias)
self.residual_gate = nn.Parameter(torch.tensor(-2.0))
self.exponent = nn.Linear(hidden_size, EXPONENT_CLASSES)
self.slot_rank = slot_rank
self.register_buffer(
"year_index",
torch.tensor(
[
[
int(f"{year:04d}"[position])
for year in range(YEAR_LOW, YEAR_HIGH + 1)
]
for position in range(YEAR_DIGITS)
],
dtype=torch.long,
),
persistent=False,
)
self.register_buffer(
"day_index", two_digit_index(range(1, 32)), persistent=False
)
self.register_buffer(
"hour_index", two_digit_index(range(24)), persistent=False
)
self.register_buffer(
"sixty_index", two_digit_index(range(60)), persistent=False
)
self.item_end = nn.Linear(hidden_size, 1)
self.bins = nn.Linear(hidden_size, bins)
self.offset = nn.Linear(hidden_size, bins)
self.overflow = nn.Linear(hidden_size, 2)
nn.init.zeros_(self.overflow.weight)
nn.init.constant_(self.overflow.bias, OVERFLOW_INIT)
nn.init.zeros_(self.offset.weight)
nn.init.zeros_(self.offset.bias)
if estimate_lexical:
self.estimate_gate = nn.Linear(hidden_size, 1)
nn.init.zeros_(self.estimate_gate.weight)
nn.init.constant_(self.estimate_gate.bias, ESTIMATE_GATE_INIT)
self.estimate_spread = nn.Linear(hidden_size, 1 + LEXICAL_DIGITS)
nn.init.zeros_(self.estimate_spread.weight)
nn.init.zeros_(self.estimate_spread.bias)
if loop_feedback:
self.loop_norm = nn.LayerNorm(hidden_size)
self.loop_down = nn.Linear(hidden_size, LOOP_RANK, bias=False)
self.loop_up = nn.Linear(LOOP_RANK, hidden_size, bias=False)
nn.init.zeros_(self.loop_up.weight)
if number_groups:
self.enable_number_groups()
if number_pointer:
self.enable_number_pointer()
edges = torch.linspace(0.0, 1.0, bins + 1)
self.register_buffer("bin_edges", edges, persistent=False)
def enable_number_groups(self) -> None:
if self.readout != "lexical":
raise ValueError("number_groups needs the lexical readout")
if self.number_groups:
return
hidden_size = self.config["hidden_size"]
device = self.norm.weight.device
self.number_group_down = nn.Linear(hidden_size, GROUP_RANK).to(device)
self.number_group_up = nn.Linear(GROUP_RANK, NUMBER_GROUPS * GROUP_CLASSES).to(
device
)
nn.init.zeros_(self.number_group_up.weight)
nn.init.zeros_(self.number_group_up.bias)
(self.number_groups,) = (True,)
self.config.update(number_groups=True)
def enable_number_pointer(self) -> None:
if self.readout != "lexical":
raise ValueError("number_pointer needs the lexical readout")
if self.number_pointer:
return
device = self.norm.weight.device
self.copy_query = nn.Linear(
self.config["hidden_size"], 2 * self.pointer_width
).to(device)
self.copy_gate = nn.Linear(self.config["hidden_size"], 1).to(device)
with torch.no_grad():
self.copy_query.weight.copy_(self.span_query.weight)
self.copy_query.bias.copy_(self.span_query.bias)
self.copy_gate.weight.zero_()
self.copy_gate.bias.fill_(COPY_GATE_INIT)
(self.number_pointer,) = (True,)
self.config.update(number_pointer=True)
def group_scores(self, states: torch.Tensor) -> torch.Tensor:
return self.number_group_up(
functional.gelu(self.number_group_down(states.float()))
).view(-1, NUMBER_GROUPS, GROUP_CLASSES)
def loop_input(self, states: torch.Tensor) -> torch.Tensor:
with torch.autocast(device_type=states.device.type, enabled=False):
return self.loop_scale * self.loop_up(
self.loop_down(self.loop_norm(states.float()))
)
@torch.no_grad()
def set_lexical_rows(self, tokenizer: Any, output_embeddings: torch.Tensor) -> None:
if self.readout != "lexical":
return
for name, words in (
("digit_rows", [str(digit) for digit in range(10)]),
("month_rows", MONTH_WORDS),
("sign_rows", SIGN_WORDS),
):
rows = []
for word in words:
ids = tokenizer(word, add_special_tokens=False)["input_ids"]
if not ids:
raise ValueError(f"tokenizer has no tokens for {word!r}")
rows.append(
output_embeddings[
torch.tensor(ids, device=output_embeddings.device)
]
.float()
.mean(dim=0)
)
getattr(self, name).copy_(torch.stack(rows).to(getattr(self, name).device))
def route(
self,
hidden: torch.Tensor,
positions: torch.Tensor,
kinds: torch.Tensor,
stop: int,
) -> tuple[torch.Tensor, torch.Tensor]:
queries = (
self.route_in(self.norm(hidden[positions].float()))
+ self.kind_embedding(kinds)
).unsqueeze(0)
memory = self.memory_in(self.token_norm(hidden[:stop].float())).unsqueeze(0)
for layer in self.route_layers:
queries = layer(queries, memory)
routed = self.route_norm(queries[0])
return (torch.sigmoid(self.route_gate) * self.route_out(routed), routed)
def answer_slot_states(
self,
hidden: torch.Tensor,
field_queries: torch.Tensor,
kinds: torch.Tensor,
positions: torch.Tensor,
lengths: torch.Tensor,
stop: int,
chunk: int = 256,
) -> torch.Tensor:
if not self.answer_slot_routing:
return hidden[positions]
if (
chunk < 1
or hidden.ndim != 2
or field_queries.ndim != 1
or (positions.ndim != 2)
or (positions.shape[0] != field_queries.numel())
):
raise ValueError("invalid answer-slot routing layout")
width = positions.shape[1]
if kinds.shape != field_queries.shape or lengths.shape != field_queries.shape:
raise ValueError("answer-slot fields, kinds and lengths must align")
if (
width < 1
or bool(((lengths < 1) | (lengths > width)).any())
or (not 0 < stop <= hidden.shape[0])
):
raise ValueError("invalid answer-slot lengths or evidence boundary")
if (
bool(((positions < 0) | (positions >= hidden.shape[0])).any())
or bool(((field_queries < 0) | (field_queries >= hidden.shape[0])).any())
or bool(((kinds < 0) | (kinds >= len(ROUTE_KINDS))).any())
):
raise ValueError("answer-slot query, token or kind out of range")
raw = hidden[positions]
if not positions.shape[0]:
return raw
output = []
with torch.autocast(device_type=hidden.device.type, enabled=False):
memory = self.memory_in(self.token_norm(hidden[:stop].float())).unsqueeze(0)
def route_chunk(
query_positions: torch.Tensor,
query_kinds: torch.Tensor,
padding: torch.Tensor,
) -> torch.Tensor:
queries = self.route_in(self.norm(hidden[query_positions].float()))
queries = queries + self.kind_embedding(query_kinds)[:, None, :]
for layer in self.route_layers:
queries = layer(
queries,
memory.expand(queries.shape[0], -1, -1),
tgt_key_padding_mask=padding,
)
return self.route_out(self.route_norm(queries[:, 1:]))
field_batch = max(1, min(chunk // (width + 1), 262144 // stop))
for first in range(0, positions.shape[0], field_batch):
last = first + field_batch
rows = positions[first:last]
query_positions = torch.cat(
[field_queries[first:last, None], rows], dim=1
)
padding = (
torch.arange(width + 1, device=hidden.device)[None, :]
> lengths[first:last, None]
)
evidence = route_chunk(query_positions, kinds[first:last], padding)
scale = self.slot_route_scale[kinds[first:last]].tanh()[:, None, None]
corrected = (
raw[first:last].float()
+ scale * torch.sigmoid(self.route_gate) * evidence
)
indices = torch.minimum(
torch.arange(width, device=hidden.device)[None, :],
lengths[first:last, None] - 1,
)
output.append(
corrected.gather(
1, indices[..., None].expand(-1, -1, corrected.shape[-1])
)
)
return torch.cat(output).to(hidden.dtype)
def typed_logits(
self, raw: torch.Tensor, routed: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
count = raw.shape[0]
scale = self.lexical_scale.clamp(max=math.log(100.0)).exp()
unit = functional.normalize(raw, dim=-1)
down = self.slot_down(raw).view(count, DIGIT_SLOTS + 2, self.slot_rank)
digit_prior = (
unit @ functional.normalize(self.digit_rows, dim=-1).t()
).unsqueeze(1)
month_prior = unit @ functional.normalize(self.month_rows, dim=-1).t()
sign_prior = unit @ functional.normalize(self.sign_rows, dim=-1).t()
residual = torch.sigmoid(self.residual_gate) * self.slot_residual(routed)
(digit_residual, month_residual, sign_residual) = residual.split(
(DIGIT_SLOTS * 10, 12, 2), dim=-1
)
return (
scale[0] * digit_prior
+ torch.einsum("nsr,srd->nsd", down[:, :DIGIT_SLOTS], self.slot_up)
+ digit_residual.view(count, DIGIT_SLOTS, 10),
scale[1] * month_prior
+ down[:, DIGIT_SLOTS] @ self.month_up
+ month_residual,
scale[2] * sign_prior
+ down[:, DIGIT_SLOTS + 1] @ self.sign_up
+ sign_residual,
)
def typed_outputs(
self, raw: torch.Tensor, states: torch.Tensor, routed: torch.Tensor
) -> dict[str, torch.Tensor]:
(digits, months, signs) = self.typed_logits(raw, routed)
year_slots = digits[:, YEAR_SLOT : YEAR_SLOT + YEAR_DIGITS]
return {
"number": torch.cat(
[signs, self.exponent(states), digits[:, :NUMBER_DIGITS].flatten(1)],
dim=-1,
),
"date": torch.cat(
[
year_slots.flatten(1),
compose_years(year_slots, self.year_index),
months,
compose_two_digits(
digits[:, DAY_SLOT : DAY_SLOT + 2], self.day_index
),
],
dim=-1,
),
"time": torch.cat(
[
compose_two_digits(
digits[:, HOUR_SLOT : HOUR_SLOT + 2], self.hour_index
),
compose_two_digits(
digits[:, MINUTE_SLOT : MINUTE_SLOT + 2], self.sixty_index
),
compose_two_digits(
digits[:, SECOND_SLOT : SECOND_SLOT + 2], self.sixty_index
),
],
dim=-1,
),
}
def forward(
self, hidden: torch.Tensor, batch: dict[str, Any]
) -> list[dict[str, Any]]:
return [
self.forward_record(hidden[index], record, batch["views"][index])
for (index, record) in enumerate(batch["records"])
]
def forward_record(
self,
hidden: torch.Tensor,
record: EncodedRecord,
views: dict[str, torch.Tensor],
) -> dict[str, Any]:
return self.forward_states(
hidden, record.input_start, record.input_start + len(record.offsets), views
)
def forward_states(
self,
hidden: torch.Tensor,
start: int,
stop: int,
views: dict[str, torch.Tensor],
) -> dict[str, Any]:
if self.readout == "lexical":
return self.forward_lexical(hidden, start, stop, views)
with torch.autocast(device_type=hidden.device.type, enabled=False):
states = self.norm(hidden[views["query_positions"]].float())
output: dict[str, Any] = {"null": self.null(states).squeeze(-1)}
tokens = None
if views["text_rows"].shape[0] or views["tags_rows"].shape[0]:
tokens = self.token_norm(hidden[start:stop].float())
if self.span_trim:
output["trim"] = self.trim(tokens).view(-1, 2, TRIM_CLASSES)
scale = 1.0 / math.sqrt(self.pointer_width)
if views["text_rows"].shape[0]:
(query_start, query_end) = self.span_query(
states[views["text_rows"]]
).split(self.pointer_width, dim=-1)
(key_start, key_end) = self.span_key(tokens).split(
self.pointer_width, dim=-1
)
output["text"] = span_scores(
query_start @ key_start.transpose(0, 1) * scale,
query_end @ key_end.transpose(0, 1) * scale,
self.length_bias,
)
if views["tags_rows"].shape[0]:
(query_begin, query_inside) = self.tag_query(
states[views["tags_rows"]]
).split(self.pointer_width, dim=-1)
keys = self.tag_key(tokens)
begin = query_begin @ keys.transpose(0, 1) * scale + self.tag_bias[0]
inside = query_inside @ keys.transpose(0, 1) * scale + self.tag_bias[1]
output["tags"] = torch.stack(
[torch.zeros_like(begin), begin, inside], dim=-1
)
for group in ("number", "date", "time"):
if views[f"{group}_rows"].shape[0]:
output[group] = getattr(self, group)(states[views[f"{group}_rows"]])
if "number" in output:
output["number"] = apply_prefix(output["number"], views, "number")
if views["estimate_rows"].shape[0]:
estimate_states = states[views["estimate_rows"]]
output["estimate"] = self.bins(estimate_states)
output["estimate_offset"] = self.offset(estimate_states)
output["estimate_overflow"] = self.overflow(estimate_states)
for kind, readout in (
("number_list", self.number),
("date_list", self.date),
):
if views[f"{kind}_rows"].shape[0]:
items = self.norm(hidden[views[f"{kind}_items"]].float())
output[f"{kind}_end"] = self.item_end(items).squeeze(-1)
output[kind] = readout(items)
return output
def forward_lexical(
self,
hidden: torch.Tensor,
start: int,
stop: int,
views: dict[str, torch.Tensor],
) -> dict[str, Any]:
with torch.autocast(device_type=hidden.device.type, enabled=False):
queries = views["query_positions"]
item_kinds = [kind for kind in LIST_KINDS if views[f"{kind}_rows"].shape[0]]
positions = torch.cat(
[queries] + [views[f"{kind}_items"].flatten() for kind in item_kinds]
)
kinds = torch.zeros(
positions.shape[0], dtype=torch.long, device=positions.device
)
for index, group in enumerate(ROUTE_KINDS):
kinds[views[f"{group}_rows"]] = index
offset = queries.shape[0]
for kind in item_kinds:
size = views[f"{kind}_items"].numel()
kinds[offset : offset + size] = len(ROUTE_KINDS) + LIST_KINDS.index(
kind
)
offset += size
(gated, routed) = self.route(hidden, positions, kinds, stop)
raw = hidden[positions].float()
states = self.norm(raw) + gated
count = queries.shape[0]
output: dict[str, Any] = {"null": self.null(states[:count]).squeeze(-1)}
(field_states, field_raw, field_routed) = (
states[:count],
raw[:count],
routed[:count],
)
tokens = None
pointed = self.number_pointer and views["number_rows"].shape[0] > 0
if views["text_rows"].shape[0] or views["tags_rows"].shape[0] or pointed:
tokens = self.token_norm(hidden[start:stop].float())
if self.span_trim and (
views["text_rows"].shape[0] or views["tags_rows"].shape[0]
):
output["trim"] = self.trim(tokens).view(-1, 2, TRIM_CLASSES)
scale = 1.0 / math.sqrt(self.pointer_width)
if pointed:
number_states = field_states[views["number_rows"]]
(query_start, query_end) = self.copy_query(number_states).split(
self.pointer_width, dim=-1
)
(key_start, key_end) = self.span_key(tokens).split(
self.pointer_width, dim=-1
)
output["number_copy"] = span_scores(
query_start @ key_start.transpose(0, 1) * scale,
query_end @ key_end.transpose(0, 1) * scale,
self.length_bias,
)
output["number_copy_gate"] = self.copy_gate(number_states).squeeze(-1)
if views["text_rows"].shape[0]:
(query_start, query_end) = self.span_query(
field_states[views["text_rows"]]
).split(self.pointer_width, dim=-1)
(key_start, key_end) = self.span_key(tokens).split(
self.pointer_width, dim=-1
)
output["text"] = span_scores(
query_start @ key_start.transpose(0, 1) * scale,
query_end @ key_end.transpose(0, 1) * scale,
self.length_bias,
)
if views["tags_rows"].shape[0]:
(query_begin, query_inside) = self.tag_query(
field_states[views["tags_rows"]]
).split(self.pointer_width, dim=-1)
keys = self.tag_key(tokens)
begin = query_begin @ keys.transpose(0, 1) * scale + self.tag_bias[0]
inside = query_inside @ keys.transpose(0, 1) * scale + self.tag_bias[1]
output["tags"] = torch.stack(
[torch.zeros_like(begin), begin, inside], dim=-1
)
typed_rows = [
views[f"{group}_rows"] for group in ("number", "date", "time")
]
if any((rows.shape[0] for rows in typed_rows)):
selected = torch.cat(typed_rows)
typed = self.typed_outputs(
field_raw[selected], field_states[selected], field_routed[selected]
)
offset = 0
for group, rows in zip(("number", "date", "time"), typed_rows):
if rows.shape[0]:
output[group] = typed[group][offset : offset + rows.shape[0]]
offset += rows.shape[0]
if "number" in output:
output["number"] = apply_prefix(output["number"], views, "number")
if self.number_groups:
output["number_groups"] = self.group_scores(
field_states[views["number_rows"]]
)
if views["estimate_rows"].shape[0]:
estimate_rows = views["estimate_rows"]
estimate_states = field_states[estimate_rows]
output["estimate"] = self.bins(estimate_states)
output["estimate_offset"] = self.offset(estimate_states)
output["estimate_overflow"] = self.overflow(estimate_states)
if self.estimate_lexical:
number_logits = self.typed_outputs(
field_raw[estimate_rows],
estimate_states,
field_routed[estimate_rows],
)["number"]
output["estimate_number_raw"] = apply_prefix(
number_logits, views, "estimate"
)
output["estimate_number"] = apply_prefix(
temper_number(
number_logits, self.estimate_spread(estimate_states)
),
views,
"estimate",
)
output["estimate_gate"] = self.estimate_gate(
estimate_states
).squeeze(-1)
offset = count
for kind in item_kinds:
shape = views[f"{kind}_items"].shape
size = shape[0] * shape[1]
(item_states, item_raw, item_routed) = (
states[offset : offset + size],
raw[offset : offset + size],
routed[offset : offset + size],
)
offset += size
output[f"{kind}_end"] = (
self.item_end(item_states).squeeze(-1).view(shape)
)
group = "number" if kind == "number_list" else "date"
output[kind] = self.typed_outputs(item_raw, item_states, item_routed)[
group
].view(*shape, -1)
if kind == "number_list" and self.number_groups:
output["number_list_groups"] = self.group_scores(item_states).view(
*shape, NUMBER_GROUPS, GROUP_CLASSES
)
return output
SPELL_END = "\n"
SPELL_SEPARATOR = " | "
SPELL_MIN_CONFIDENCE = 0.3
def spelled_answer(
ids: list[int], log_probs: list[float], tokenizer: Any, end_id: int
) -> tuple[str, float, list[tuple[str, float]]]:
stop = ids.index(end_id) if end_id in ids else len(ids)
pieces = [tokenizer.decode([token]) for token in ids[:stop]]
items: list[tuple[list[int], list[float]]] = []
current: list[int] = []
scores: list[float] = []
for token, piece, score in zip(ids[:stop], pieces, log_probs[:stop]):
if piece.strip() == SPELL_SEPARATOR.strip():
items.append((current, scores))
(current, scores) = ([], [])
else:
current.append(token)
scores.append(score)
items.append((current, scores))
parts = []
for item_ids, item_scores in items:
words = tokenizer.decode(item_ids).split()
if words:
parts.append(
(" ".join(words), math.exp(sum(item_scores) / len(item_scores)))
)
text = SPELL_SEPARATOR.join((part for (part, _) in parts))
return (text, math.exp(sum(log_probs[: min(stop + 1, len(ids))])), parts)
def evidence_text_answer(
prediction: dict[str, Any],
ids: list[int],
log_probs: list[float],
tokenizer: Any,
end_id: int,
text: str,
spec: dict[str, Any],
has_images: bool = False,
) -> dict[str, Any]:
if end_id not in ids:
return prediction
stop = ids.index(end_id)
value = tokenizer.decode(ids[:stop]).strip()
confidence = math.exp(sum(log_probs[: stop + 1]))
exact = (
[
span
for span in zs.find_spans(text, value)
if text[span[0] : span[1]] == value
]
if value
else []
)
present = prediction["p_null"] < zs.null_threshold(spec)
return {
**prediction,
"guess": value or None,
"value": value if value and present else None,
"span": list(exact[0]) if exact else None,
"confidence": confidence,
"source": "input" if exact else "image" if has_images else "readout",
}
def _simple(value: Any) -> str:
return "".join(
(character for character in str(value).lower() if character.isalnum())
)
def _similarity(left: Any, right: Any) -> float:
import difflib
(left, right) = (_simple(left), _simple(right))
return difflib.SequenceMatcher(None, left, right).ratio() if left and right else 0.0
def _state_match(
text: str, value: str, threshold: float = 0.8, max_words: int = 4000
) -> tuple[int, int] | None:
if not text or not value:
return None
spans = zs.find_spans(text, value)
if spans:
return spans[0]
import re
words = [match.span() for match in re.finditer("\\S+", text)][:max_words]
size = len(value.split())
(best, best_span) = (threshold, None)
for width in range(max(1, size - 1), size + 2):
for first in range(max(0, len(words) - width + 1)):
(start, end) = (words[first][0], words[first + width - 1][1])
score = _similarity(text[start:end].strip(".,;:()[]\"'"), value)
if score >= best:
(best, best_span) = (score, (start, end))
return best_span
def _stutters(value: Any) -> bool:
import re
simple = _simple(value)
return bool(re.search("(.)\\1\\1", simple) or re.search("([a-z]{4,})\\1", simple))
def _digit_code(value: Any) -> bool:
simple = _simple(value)
return bool(simple) and 2 * sum(
(character.isdigit() for character in simple)
) >= len(simple)
PLACEHOLDER_WORDS = frozenset(
(
"a",
"above",
"an",
"app",
"attach",
"attached",
"attachment",
"below",
"doc",
"document",
"documents",
"file",
"heic",
"here",
"image",
"images",
"is",
"jpeg",
"jpg",
"mobile",
"page",
"pdf",
"photo",
"photos",
"pic",
"picture",
"please",
"png",
"scan",
"scanned",
"screenshot",
"see",
"the",
"this",
"upload",
"uploaded",
"via",
)
)
def _placeholder_text(value: Any) -> bool:
import re
words = re.findall("[a-z0-9]+", str(value or "").lower())
return bool(words) and all((word in PLACEHOLDER_WORDS for word in words))
def _defers(pointer_value: Any, spelled: str) -> bool:
if pointer_value is None:
return False
return _stutters(spelled) or (
_digit_code(spelled)
and any((character.isdigit() for character in str(pointer_value)))
)
def route_spelled(
prediction: dict[str, Any],
kind: str,
spelled: str,
items: list[tuple[str, float]],
text: str,
defer_lists: bool = False,
) -> dict[str, Any]:
if kind == "text":
if _placeholder_text(prediction.get("guess")):
prediction = {
**prediction,
"value": None,
"guess": None,
"span": None,
"confidence": 0.0,
}
items = [
(item, score) for (item, score) in items if not _placeholder_text(item)
]
if not items:
return prediction
(value, confidence) = (
items[0][0] if len(items) == 1 else spelled,
min((score for (_, score) in items)),
)
routed = {**prediction, "spell_confidence": confidence}
if _similarity(prediction.get("guess"), value) >= 0.6:
if (
routed.get("value") is None
and prediction.get("guess")
and (confidence >= SPELL_MIN_CONFIDENCE)
):
return {**routed, "value": prediction["guess"]}
return routed
span = _state_match(text, value)
if span is not None:
return {
**routed,
"value": text[span[0] : span[1]],
"guess": text[span[0] : span[1]],
"span": list(span),
}
if confidence >= SPELL_MIN_CONFIDENCE and (
not _defers(prediction.get("value"), value)
):
return {
**routed,
"value": value,
"guess": value,
"span": None,
"confidence": confidence,
"source": "image",
}
return routed
(pointer_values, pointer_spans) = (
list(prediction.get("value") or []),
list(prediction.get("spans") or []),
)
if any((_placeholder_text(value) for value in pointer_values)):
kept = [
(value, pointer_spans[index] if index < len(pointer_spans) else None)
for (index, value) in enumerate(pointer_values)
if not _placeholder_text(value)
]
(pointer_values, pointer_spans) = (
[value for (value, _) in kept],
[span for (_, span) in kept],
)
items = [(item, score) for (item, score) in items if not _placeholder_text(item)]
(values, spans, used) = ([], [], set())
for item, confidence in items:
match = max(
range(len(pointer_values)),
key=lambda index: _similarity(pointer_values[index], item),
default=None,
)
if (
match is not None
and match not in used
and (_similarity(pointer_values[match], item) >= 0.6)
):
used.add(match)
values.append(pointer_values[match])
spans.append(pointer_spans[match] if match < len(pointer_spans) else None)
continue
span = _state_match(text, item)
if span is not None:
value = text[span[0] : span[1]]
if not any((_simple(value) == _simple(other) for other in values)):
values.append(value)
spans.append(list(span))
elif (
confidence >= SPELL_MIN_CONFIDENCE
and (
not (
pointer_values and (defer_lists or _defers(pointer_values[0], item))
)
)
and (not any((_simple(item) == _simple(other) for other in values)))
):
values.append(item)
spans.append(None)
for index, (value, span) in enumerate(zip(pointer_values, pointer_spans)):
if index not in used and (
not any((_simple(value) == _simple(other) for other in values))
):
values.append(value)
spans.append(span)
routed = {**prediction, "value": values, "spans": spans}
if any((span is None for span in spans)):
routed["source"] = "image"
return routed
def number_parts(
logits: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
(sign, exponent, digits) = logits.split(
(2, EXPONENT_CLASSES, NUMBER_DIGITS * 10), dim=-1
)
return (sign, exponent, digits.unflatten(-1, (NUMBER_DIGITS, 10)))
def group_class_digits(device: Any = None) -> torch.Tensor:
classes = torch.arange(GROUP_CLASSES, device=device)
return torch.stack(
[
classes // 10 ** (GROUP_DIGITS - 1 - place) % 10
for place in range(GROUP_DIGITS)
],
dim=-1,
)
def group_slots(exponent_class: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
exponent_class = exponent_class.long()
device = exponent_class.device
power = (exponent_class - 1 + MIN_EXPONENT).view(-1, 1, 1)
groups = torch.arange(GROUP_LOW, GROUP_HIGH + 1, device=device).view(1, -1, 1)
places = torch.arange(GROUP_DIGITS - 1, -1, -1, device=device).view(1, 1, -1)
slots = power - (groups * GROUP_DIGITS + places)
inside = (
(slots >= 0) & (slots < NUMBER_DIGITS) & (exponent_class > 0).view(-1, 1, 1)
)
return (
torch.where(inside, slots, torch.full_like(slots, NUMBER_DIGITS)),
inside.any(dim=-1),
)
def number_group_logits(
digit_logits: torch.Tensor,
exponent_class: torch.Tensor,
learned: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
(count, device) = (digit_logits.shape[0], digit_logits.device)
leading_zero = torch.zeros(NUMBER_DIGITS, 10, dtype=torch.bool, device=device)
leading_zero[0, 0] = True
log = torch.log_softmax(digit_logits.float(), dim=-1).masked_fill(
leading_zero, MASKED
)
held = torch.full((count, 1, 10), MASKED, device=device, dtype=log.dtype)
held[..., 0] = 0.0
padded = torch.cat([log, held], dim=1)
(slots, groups) = group_slots(exponent_class.to(device))
picked = padded.gather(1, slots.flatten(1).unsqueeze(-1).expand(-1, -1, 10)).view(
count, NUMBER_GROUPS, GROUP_DIGITS, 10
)
table = group_class_digits(device)
scores = cast(
torch.Tensor,
sum(
(
picked[:, :, place, :].index_select(-1, table[:, place])
for place in range(GROUP_DIGITS)
)
),
)
if learned is not None:
scores = scores + learned.float()
return (scores, groups)
def temper_number(logits: torch.Tensor, spread: torch.Tensor) -> torch.Tensor:
(sign, exponent, digits) = number_parts(logits)
temperature = torch.exp(
ESTIMATE_SPREAD_LIMIT * torch.tanh(spread.float() / ESTIMATE_SPREAD_LIMIT)
)
exponent = exponent / temperature[:, :1]
digits = torch.cat(
[
digits[:, :LEXICAL_DIGITS] / temperature[:, 1:].unsqueeze(-1),
digits[:, LEXICAL_DIGITS:],
],
dim=1,
)
return torch.cat([sign, exponent, digits.flatten(1)], dim=-1)
def apply_prefix(
logits: torch.Tensor, views: dict[str, torch.Tensor], group: str
) -> torch.Tensor:
length = views.get(f"{group}_prefix_length")
if length is None or not logits.shape[0]:
return logits
device = logits.device
(sign_logits, exponent_logits, digit_logits) = number_parts(logits)
slots = torch.arange(NUMBER_DIGITS, device=device)
length = length.long()
relative = (slots.unsqueeze(0) - length.unsqueeze(1)).clamp_min(0)
shifted = digit_logits.gather(1, relative.unsqueeze(-1).expand(-1, -1, 10))
forced_digits = torch.where(
torch.arange(10, device=device)
== views[f"{group}_prefix_digits"].long().unsqueeze(-1),
0.0,
MASKED,
).to(logits.dtype)
digit_logits = torch.where(
(slots.unsqueeze(0) < length.unsqueeze(1)).unsqueeze(-1), forced_digits, shifted
)
written = (length > 0).unsqueeze(1)
forced_sign = torch.where(
torch.arange(2, device=device)
== views[f"{group}_prefix_sign"].long().unsqueeze(1),
0.0,
MASKED,
).to(logits.dtype)
forced_exponent = torch.where(
torch.arange(EXPONENT_CLASSES, device=device)
== views[f"{group}_prefix_exponent"].long().unsqueeze(1),
0.0,
MASKED,
).to(logits.dtype)
return torch.cat(
[
torch.where(written, forced_sign, sign_logits),
torch.where(written, forced_exponent, exponent_logits),
digit_logits.flatten(1),
],
dim=-1,
)
def window_log_probs(
log_probs: torch.Tensor, window: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
bins = torch.arange(log_probs.shape[-1], device=log_probs.device)
window = window.to(log_probs.device).long()
inside = (bins >= window[:, :1]) & (bins <= window[:, 1:])
return (torch.log_softmax(log_probs.masked_fill(~inside, MASKED), dim=-1), inside)
def date_parts(
logits: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
(digits, year, month, day) = logits.split(
(YEAR_DIGITS * 10, YEAR_CLASSES, 12, 31), dim=-1
)
return (digits.unflatten(-1, (YEAR_DIGITS, 10)), year, month, day)
def time_parts(logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return logits.split((24, 60, 60), dim=-1)
def lexical_bin_log_probs(
number_logits: torch.Tensor,
edge_sign: torch.Tensor,
edge_class: torch.Tensor,
edge_digits: torch.Tensor,
edge_frac: torch.Tensor,
) -> torch.Tensor:
(sign, exponent, digits) = number_parts(number_logits.double())
sign_probs = torch.softmax(sign, dim=-1)
exponent_probs = torch.softmax(exponent, dim=-1)
zero = exponent_probs[:, :1]
nonzero = exponent_probs[:, 1:] / (1.0 - zero).clamp_min(1e-300)
below_class = torch.cumsum(nonzero, dim=-1) - nonzero
leading = digits[:, :1].masked_fill(
torch.arange(10, device=digits.device) == 0, float("-inf")
)
digit_probs = torch.softmax(
torch.cat([leading, digits[:, 1:LEXICAL_DIGITS]], dim=1), dim=-1
)
below_digit = torch.cumsum(digit_probs, dim=-1) - digit_probs
classes = (edge_class - 1).clamp(0, nonzero.shape[1] - 1)
within = edge_frac.double()
for position in reversed(range(LEXICAL_DIGITS)):
index = edge_digits[..., position]
within = (
below_digit[:, position].gather(1, index)
+ digit_probs[:, position].gather(1, index) * within
)
magnitude = below_class.gather(1, classes) + nonzero.gather(1, classes) * within
negative = sign_probs[:, 1:] * (1.0 - zero)
positive = sign_probs[:, :1] * (1.0 - zero)
cdf = torch.where(
edge_sign == EDGE_NEGATIVE,
negative * (1.0 - magnitude),
torch.where(
edge_sign == EDGE_ZERO,
negative.expand_as(magnitude),
negative + zero + positive * magnitude,
),
)
mass = (cdf[:, 1:] - cdf[:, :-1]).clamp_min(0.0) + 1e-30
return (mass.log() - mass.sum(dim=-1, keepdim=True).log()).float()
def estimate_log_probs(
range_logits: torch.Tensor,
number_logits: torch.Tensor | None = None,
gate: torch.Tensor | None = None,
lexical: torch.Tensor | None = None,
edge_sign: torch.Tensor | None = None,
edge_class: torch.Tensor | None = None,
edge_digits: torch.Tensor | None = None,
edge_frac: torch.Tensor | None = None,
) -> torch.Tensor:
range_part = torch.log_softmax(range_logits.float(), dim=-1)
if number_logits is None or not number_logits.shape[0]:
return range_part
lexical_part = lexical_bin_log_probs(
number_logits,
cast(torch.Tensor, edge_sign),
cast(torch.Tensor, edge_class),
cast(torch.Tensor, edge_digits),
cast(torch.Tensor, edge_frac),
)
use = cast(torch.Tensor, lexical).bool().unsqueeze(1)
gate = cast(torch.Tensor, gate).float().unsqueeze(1)
mixed = torch.logaddexp(
functional.logsigmoid(gate) + lexical_part,
functional.logsigmoid(-gate) + range_part,
)
return torch.where(use, mixed, range_part)
def decode_number(
logits: torch.Tensor, spec: dict[str, Any], groups: torch.Tensor | None = None
) -> tuple[float | int, float]:
(sign_logits, exponent_logits, digit_logits) = number_parts(logits.float().cpu())
decimals = zs.allowed_decimals(spec)
exponent_log = torch.log_softmax(exponent_logits, dim=-1)
exponent_class = int(exponent_log.argmax())
log_probability = float(exponent_log[exponent_class])
if exponent_class == 0:
return (0, math.exp(log_probability))
power = exponent_class - 1 + MIN_EXPONENT
kept = NUMBER_DIGITS if decimals is None else power + decimals + 1
if kept < 0:
return (0, math.exp(log_probability))
leading_zero = torch.zeros(NUMBER_DIGITS, 10, dtype=torch.bool)
leading_zero[0, 0] = True
digit_log = torch.log_softmax(digit_logits, dim=-1).masked_fill(
leading_zero, MASKED
)
digits = digit_log.argmax(dim=-1).tolist()
sign_log = torch.log_softmax(sign_logits, dim=-1)
sign = int(sign_log.argmax())
kept = min(kept, NUMBER_DIGITS)
log_probability += float(sign_log[sign])
if groups is None:
log_probability += sum(
(float(digit_log[position, digits[position]]) for position in range(kept))
)
if kept < NUMBER_DIGITS:
rounding = digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5]
log_probability += float(torch.logsumexp(rounding, dim=0))
else:
exponent_tensor = torch.tensor([exponent_class])
(scores, present) = number_group_logits(
digit_logits.unsqueeze(0),
exponent_tensor,
groups.float().cpu().unsqueeze(0),
)
group_log = torch.log_softmax(scores[0], dim=-1)
slots = group_slots(exponent_tensor)[0][0].tolist()
table = group_class_digits()
counted = set()
for group in range(NUMBER_GROUPS):
if not bool(present[0, group]):
continue
best = int(group_log[group].argmax())
for place, slot in enumerate(slots[group]):
if slot < NUMBER_DIGITS:
digits[slot] = int(table[best, place])
if any((slot < kept for slot in slots[group])):
counted.add(group)
log_probability += float(group_log[group, best])
if kept < NUMBER_DIGITS:
for group in range(NUMBER_GROUPS):
if kept in slots[group]:
if group not in counted:
place = slots[group].index(kept)
same = (
table[:, place] >= 5
if digits[kept] >= 5
else table[:, place] < 5
)
log_probability += float(
torch.logsumexp(group_log[group][same], dim=0)
)
break
else:
rounding = (
digit_log[kept, 5:] if digits[kept] >= 5 else digit_log[kept, :5]
)
log_probability += float(torch.logsumexp(rounding, dim=0))
magnitude = Decimal(int("".join((str(digit) for digit in digits)))).scaleb(
power - NUMBER_DIGITS + 1
)
if decimals is not None:
magnitude = magnitude.quantize(
Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP
)
if magnitude == 0:
return (0, math.exp(log_probability))
value = -magnitude if sign else magnitude
if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53):
return (int(value), math.exp(log_probability))
return (float(value), math.exp(log_probability))
def number_from_numeral(
magnitude: Decimal,
logits: torch.Tensor,
spec: dict[str, Any],
check_exponent: bool = True,
) -> float | int | None:
(sign_logits, exponent_logits, _) = number_parts(logits.float().cpu())
if check_exponent:
target = number_components(float(magnitude))
if target is None or int(exponent_logits.argmax()) != target[1]:
return None
decimals = zs.allowed_decimals(spec)
if decimals is not None:
magnitude = magnitude.quantize(
Decimal(1).scaleb(-decimals), rounding=ROUND_HALF_UP
)
if magnitude == 0:
return 0
value = -magnitude if int(sign_logits.argmax()) else magnitude
if decimals == 0 or (value == value.to_integral_value() and abs(value) < 2**53):
return int(value)
return float(value)
def number_from_numerals(
values: list[Decimal],
logits: torch.Tensor,
spec: dict[str, Any],
readout: float | None,
confident: bool = False,
) -> float | int | None:
agreeing = [
value
for value in (
number_from_numeral(magnitude, logits, spec) for magnitude in values
)
if value is not None
]
if not agreeing and confident and values:
def distance(magnitude: Decimal) -> float:
if readout is None or not readout or (not magnitude):
return 0.0
return abs(math.log10(float(magnitude)) - math.log10(abs(float(readout))))
agreeing = [
value
for value in (
number_from_numeral(min(values, key=distance), logits, spec, False),
)
if value is not None
]
if not agreeing:
return None
if readout is None:
return agreeing[0]
return min(agreeing, key=lambda value: abs(float(value) - float(readout)))
def copied_number(
output: dict[str, Any],
row: int,
record: EncodedRecord,
spec: dict[str, Any],
readout: float | None = None,
) -> tuple[float | int, float, list[int]] | None:
if (
"number_copy_gate" not in output
or record.record.get("images")
or (not record.text)
):
return None
if os.environ.get("SZ_DISABLE_NUMBER_COPY") == "1":
return None
gate = float(
torch.sigmoid(torch.as_tensor(output["number_copy_gate"][row]).float())
)
if gate < 0.5:
return None
if "number_copy_best" in output:
(best, probability) = (
output["number_copy_best"][row],
output["number_copy_probability"][row],
)
else:
scores = output["number_copy"][row].float().flatten()
(best, probability) = (
int(scores.argmax()),
float(torch.softmax(scores, dim=0).max()),
)
(start, length) = divmod(int(best), MAX_SPAN)
last = min(start + length, len(record.offsets) - 1)
if start >= len(record.offsets):
return None
(char_start, char_end) = (record.offsets[start][0], record.offsets[last][1])
while char_start < char_end and (not record.text[char_start].isalnum()):
char_start += 1
around = [
found
for found in read_numerals(record.text, every=True)
if found[1] < char_end and found[2] > char_start
]
if not around:
return None
nearest = min(
around,
key=lambda found: (
abs(found[1] - char_start) + abs(found[2] - char_end),
-(found[2] - found[1]),
),
)
whole = [
found
for found in read_numerals(record.text)
if found[1] <= char_start
and found[2] >= char_end
and (found[2] - found[1] > nearest[2] - nearest[1])
]
choices = whole[:1] + [nearest]
if gate * probability < COPY_MIN_CONFIDENCE:
return None
sure = gate >= COPY_CONFIDENT
(value, numeral_start, numeral_end) = (None, nearest[1], nearest[2])
for values, start, end in choices:
run = NUMERAL_RUN.match(record.text, start)
printed = record.text[start : min(end, run.end())] if run else ""
settled = (
numeral_convention(record.text) is not None
and len(numeral_values(printed)) > 1
)
enclosing = bool(whole) and (start, end) == (whole[0][1], whole[0][2])
value = number_from_numerals(
values,
output["number"][row],
spec,
readout,
confident=settled or (sure and enclosing and (len(values) == 1)),
)
if value is not None:
(numeral_start, numeral_end) = (start, end)
break
if value is None and sure:
(values, numeral_start, numeral_end) = choices[0]
value = number_from_numerals(
values, output["number"][row], spec, readout, confident=True
)
if value is None or (readout is not None and rescaled(readout, value)):
return None
return (value, gate * probability, [numeral_start, numeral_end])
def rescaled(readout: float, copied: float) -> bool:
try:
(first, second) = (
abs(Decimal(str(readout))).normalize(),
abs(Decimal(str(copied))).normalize(),
)
except ArithmeticError:
return False
if not first or not second or first == second:
return False
return first.as_tuple().digits == second.as_tuple().digits
def decode_date(logits: torch.Tensor) -> tuple[str, float]:
(digit_logits, year_logits, month_logits, day_logits) = date_parts(
logits.float().cpu()
)
class_log = torch.log_softmax(year_logits, dim=-1)
year_class = int(class_log.argmax())
log_probability = float(class_log[year_class])
if year_class < YEAR_OUTSIDE:
year = YEAR_LOW + year_class
else:
digit_log = torch.log_softmax(digit_logits, dim=-1)
digits = digit_log.argmax(dim=-1).tolist()
if digits == [0, 0, 0, 0]:
masked = digit_log[3].clone()
masked[0] = MASKED
digits[3] = int(masked.argmax())
year = int("".join((str(digit) for digit in digits)))
log_probability += float(
sum((digit_log[position, digit] for (position, digit) in enumerate(digits)))
)
(month_log, day_log) = (
torch.log_softmax(month_logits, dim=-1),
torch.log_softmax(day_logits, dim=-1),
)
joint = month_log.unsqueeze(1) + day_log.unsqueeze(0)
lengths = torch.tensor(
[calendar.monthrange(year, month)[1] for month in range(1, 13)]
)
joint = joint.masked_fill(
torch.arange(31).unsqueeze(0) >= lengths.unsqueeze(1), MASKED
)
best = int(joint.flatten().argmax())
(month, day) = divmod(best, 31)
log_probability += float(joint.flatten()[best])
return (
datetime.date(year, month + 1, day + 1).isoformat(),
math.exp(log_probability),
)
def decode_time(logits: torch.Tensor) -> tuple[str, float]:
parts = [
torch.log_softmax(part, dim=-1) for part in time_parts(logits.float().cpu())
]
values = [int(part.argmax()) for part in parts]
return (
zs.format_time(*values),
math.exp(sum((float(part[value]) for (part, value) in zip(parts, values)))),
)
def decode_spans(
labels: list[int],
offsets: list[tuple[int, int]],
text: str,
trims: list[list[int]] | None = None,
) -> tuple[list[str], list[list[int]]]:
spans: list[list[int]] = []
current: list[int] | None = None
for position, label in enumerate(labels):
if label == TAG_BEGIN:
if current is not None:
spans.append(current)
current = [position, position]
elif label == TAG_INSIDE and current is not None:
current[1] = position
elif current is not None:
spans.append(current)
current = None
if current is not None:
spans.append(current)
values: list[str] = []
char_spans: list[list[int]] = []
for first, last in spans:
(char_start, char_end) = trimmed_span(text, offsets, first, last, trims)
if not text[char_start:char_end].strip():
continue
char_spans.append([char_start, char_end])
if text[char_start:char_end] not in values:
values.append(text[char_start:char_end])
return (values, char_spans)
def quantile_from_cdf(
cdf: list[float], probabilities: list[float], fraction: float, bins: int
) -> float:
index = min(bisect_left(cdf, fraction), bins - 1)
previous = cdf[index - 1] if index > 0 else 0.0
inside = (
(fraction - previous) / probabilities[index]
if probabilities[index] > 0
else 0.5
)
return (index + min(1.0, max(0.0, inside))) / bins
def decode_estimate(
logits: torch.Tensor,
offsets: torch.Tensor,
overflow: torch.Tensor,
spec: dict[str, Any],
) -> dict[str, Any]:
probabilities = torch.softmax(logits.float(), dim=-1)
cdf = torch.cumsum(probabilities, dim=-1).tolist()
row = probabilities.tolist()
bins = len(row)
shift = torch.sigmoid(offsets.float()).tolist()
(low, high, scale) = zs.estimate_range(spec)
centers = [(index + shift[index]) / bins for index in range(bins)]
mean = sum(
(
probability * zs.from_unit(center, low, high, scale)
for (probability, center) in zip(row, centers)
)
)
quantiles: list[Any] = [
zs.from_unit(
quantile_from_cdf(cdf, row, fraction / 100.0, bins), low, high, scale
)
for fraction in range(1, 100)
]
value = (
zs.from_unit(
sum((probability * center for (probability, center) in zip(row, centers))),
low,
high,
scale,
)
if scale == "log"
else mean
)
step = zs.estimate_step(spec)
if step > 0:
quantiles = [
zs.snap_quantile(quantile, step, low, high) for quantile in quantiles
]
value = zs.snap(mean, step, low, high)
if zs.is_date_estimate(spec):
(value, mean) = (zs.estimate_value(spec, value), zs.estimate_value(spec, mean))
quantiles = [zs.estimate_value(spec, quantile) for quantile in quantiles]
prediction = {
"value": value,
"mean": mean,
"median": quantiles[49],
"p10": quantiles[9],
"p90": quantiles[89],
"quantiles": quantiles,
}
if not zs.is_date_estimate(spec):
sides = torch.softmax(
torch.cat([overflow.new_zeros(1), overflow.float()]), dim=0
).tolist()
(prediction["p_below"], prediction["p_above"]) = (sides[BELOW], sides[ABOVE])
return prediction
def estimate_row_log_probs(
output: dict[str, Any],
row: int,
spec: dict[str, Any],
window: tuple[int, int] | None = None,
) -> torch.Tensor:
log_probs = _estimate_row_log_probs(output, row, spec)
if window is None:
return log_probs
return window_log_probs(log_probs.unsqueeze(0), torch.tensor([window]))[0][0]
def _estimate_row_log_probs(
output: dict[str, Any], row: int, spec: dict[str, Any]
) -> torch.Tensor:
logits = output["estimate"][row].float().cpu()
parts = (
estimate_edge_parts(spec, logits.shape[-1])
if "estimate_number" in output
else None
)
if parts is None:
return torch.log_softmax(logits, dim=-1)
(sign, classes, digits, fractions) = parts
count = len(sign)
return estimate_log_probs(
logits.unsqueeze(0),
output["estimate_number"][row].float().cpu().unsqueeze(0),
output["estimate_gate"][row].float().cpu().reshape(1),
torch.ones(1),
torch.tensor([sign]),
torch.tensor([classes]),
torch.tensor(digits).view(1, count, LEXICAL_DIGITS),
torch.tensor([fractions], dtype=torch.float64),
)[0]
def decode_record(
output: dict[str, Any],
record: EncodedRecord,
tokenizer: Any = None,
end_id: int | None = None,
) -> dict[str, dict[str, Any]]:
fields = record.fields
specs = {item.name: record.record["fields"][item.name] for item in fields}
p_null = torch.sigmoid(output["null"].float()).tolist()
predictions: dict[str, dict[str, Any]] = {}
value: Any
rows_of = {
group: [
position for (position, item) in enumerate(fields) if item.kind in kinds
]
for (group, kinds) in (
("text", ("text",)),
("tags", ("text_list",)),
("number", ("number",)),
("date", ("date", "datetime")),
("time", ("time", "datetime")),
("estimate", ("estimate",)),
("number_list", ("number_list",)),
("date_list", ("date_list",)),
)
}
index_of = {
group: {position: row for (row, position) in enumerate(rows)}
for (group, rows) in rows_of.items()
}
if rows_of["text"] and "text_best" in output:
(best, probabilities) = (output["text_best"], output["text_probability"])
elif rows_of["text"]:
scores = output["text"].float()
best = scores.flatten(1).argmax(dim=1).tolist()
probabilities = torch.softmax(scores.flatten(1), dim=1).amax(dim=1).tolist()
if rows_of["tags"]:
labels = (
output["tags_labels"]
if "tags_labels" in output
else output["tags"].argmax(dim=-1).tolist()
)
trims = output.get("trim_labels")
if trims is None and "trim" in output:
trims = output["trim"].argmax(dim=-1).tolist()
for position, item in enumerate(fields):
spec = specs[item.name]
kind = item.kind
if kind == "estimate":
row = index_of["estimate"][position]
predictions[item.name] = decode_estimate(
estimate_row_log_probs(output, row, spec, item.window),
output["estimate_offset"][row],
output["estimate_overflow"][row],
spec,
)
continue
if kind == "text_list":
row = index_of["tags"][position]
(values, spans) = decode_spans(
labels[row], record.offsets, record.text, trims
)
predictions[item.name] = {"value": values, "spans": spans}
if "tags_log_probability" in output:
predictions[item.name]["confidence"] = math.exp(
output["tags_log_probability"][row]
)
elif "tags" in output:
label_log = torch.log_softmax(output["tags"][row].float().cpu(), dim=-1)
predictions[item.name]["confidence"] = math.exp(
float(label_log.amax(dim=-1).sum())
)
continue
if kind in LIST_KINDS:
row = index_of[kind][position]
ends = torch.sigmoid(output[f"{kind}_end"][row].float()).tolist()
(values, confidences) = ([], [])
log_probability = 0.0
for slot in range(LIST_SLOTS):
if ends[slot] >= 0.5:
log_probability += math.log(max(ends[slot], 1e-12))
break
log_probability += math.log(max(1.0 - ends[slot], 1e-12))
if kind == "number_list":
(value, confidence) = decode_number(
output[kind][row, slot],
spec,
output["number_list_groups"][row, slot]
if "number_list_groups" in output
else None,
)
else:
(value, confidence) = decode_date(output[kind][row, slot])
values.append(value)
confidences.append(confidence)
log_probability += math.log(max(confidence, 1e-12))
predictions[item.name] = {
"value": values,
"confidences": confidences,
"confidence": math.exp(log_probability),
}
continue
prediction: dict[str, Any] = {"p_null": p_null[position]}
if kind == "text":
row = index_of["text"][position]
(start, length) = divmod(best[row], MAX_SPAN)
last = min(start + length, len(record.offsets) - 1)
(char_start, char_end) = trimmed_span(
record.text, record.offsets, start, last, trims
)
(value, confidence) = (record.text[char_start:char_end], probabilities[row])
prediction["span"] = [char_start, char_end]
if not record.text:
(value, prediction["span"]) = (None, None)
elif kind == "number":
number_row = index_of["number"][position]
(value, confidence) = decode_number(
output["number"][number_row],
spec,
output["number_groups"][number_row]
if "number_groups" in output
else None,
)
copied = copied_number(output, number_row, record, spec, value)
if (
copied is not None
and value is not None
and (os.environ.get("SZ_NUMBER_COPY_VETO", "1") == "1")
and numeral_spans(record.text, value, every=False)
):
copied = None
if copied is not None:
(value, confidence, prediction["span"]) = copied
prediction["source"] = "input"
if spec["type"] == "duration":
prediction["unit"] = zs.duration_unit(spec)
elif kind == "date":
(value, confidence) = decode_date(
output["date"][index_of["date"][position]]
)
elif kind == "time":
(value, confidence) = decode_time(
output["time"][index_of["time"][position]]
)
else:
(day, day_confidence) = decode_date(
output["date"][index_of["date"][position]]
)
(clock, clock_confidence) = decode_time(
output["time"][index_of["time"][position]]
)
(value, confidence) = (f"{day}T{clock}", day_confidence * clock_confidence)
prediction["guess"] = value
prediction["confidence"] = confidence
prediction["value"] = (
None if p_null[position] >= zs.null_threshold(spec) else value
)
predictions[item.name] = prediction
if tokenizer is not None and end_id is not None and ("spell_ids" in output):
(ids, log_probs) = (
output["spell_ids"].tolist(),
output["spell_log_probs"].float().tolist(),
)
for position, item in enumerate(fields):
if item.kind == "text":
row = index_of["text"][position]
elif item.kind == "text_list":
row = len(rows_of["text"]) + index_of["tags"][position]
else:
continue
(spelled, _, parts) = spelled_answer(
ids[row], log_probs[row], tokenizer, end_id
)
predictions[item.name] = route_spelled(
predictions[item.name], item.kind, spelled, parts, record.text
)
if tokenizer is not None and "slot_ids" in output:
slot_end = _tokens(tokenizer, SPELL_END)[0]
(slot_ids, slot_log_probs) = (
output["slot_ids"].tolist(),
output["slot_log_probs"].float().tolist(),
)
slot_rows = [
position for (position, item) in enumerate(fields) if item.answer_slots
]
for row, position in enumerate(slot_rows):
item = fields[position]
if item.kind not in ("text", "text_list"):
continue
count = len(item.answer_slots)
(spelled, _probability, parts) = spelled_answer(
slot_ids[row][:count], slot_log_probs[row][:count], tokenizer, slot_end
)
prediction = predictions[item.name]
if (record.record.get("_meta") or {}).get(
"construction"
) == zs.EVIDENCE_CONSTRUCTION and item.kind == "text":
predictions[item.name] = evidence_text_answer(
prediction,
slot_ids[row][:count],
slot_log_probs[row][:count],
tokenizer,
slot_end,
record.text,
specs[item.name],
bool(record.record.get("images")),
)
continue
if item.kind in ("text", "text_list"):
predictions[item.name] = route_spelled(
prediction, item.kind, spelled, parts, record.text, defer_lists=True
)
continue
return predictions
FIELD_KINDS = SCALAR_KINDS + ("estimate", "text_list") + LIST_KINDS
VIEW_GROUPS = (
("text", ("text",)),
("tags", ("text_list",)),
("number", ("number",)),
("date", ("date", "datetime")),
("time", ("time", "datetime")),
("estimate", ("estimate",)),
)
COMPACT_KEYS = (
"number",
"date",
"time",
"estimate",
"estimate_offset",
"estimate_overflow",
"number_list_end",
"number_list",
"date_list_end",
"date_list",
"estimate_number",
"estimate_gate",
"number_groups",
"number_list_groups",
"number_copy_gate",
)
COMPACT_GROUPS = {
"number": "number",
"date": "date",
"time": "time",
"estimate": "estimate",
"estimate_offset": "estimate",
"estimate_overflow": "estimate",
"number_list_end": "number_list",
"number_list": "number_list",
"date_list_end": "date_list",
"date_list": "date_list",
"estimate_number": "estimate",
"estimate_gate": "estimate",
"number_groups": "number",
"number_list_groups": "number_list",
"number_copy_gate": "number",
}
def prediction_layout(record: EncodedRecord) -> list[int]:
layout = [
len(record.input_ids),
record.input_start,
len(record.offsets),
len(record.fields),
]
for item in record.fields:
layout += [
FIELD_KINDS.index(item.kind),
item.query,
len(item.items),
*item.items,
]
slotted = [
(index, item.answer_slots)
for (index, item) in enumerate(record.fields)
if item.answer_slots
]
if slotted:
layout.append(len(slotted))
for index, slots in slotted:
layout += [index, len(slots), *slots]
return layout
def parse_prediction_layout(
layout: list[int], num_tokens: int, with_slots: bool = False
) -> tuple[Any, ...]:
if len(layout) < 4 or any((type(value) is not int for value in layout)):
raise ValueError("prediction_layout must be a list of ints")
(prompt_length, input_start, input_length, count) = layout[:4]
shift = num_tokens - prompt_length
if (
shift < 0
or count < 1
or input_length < 1
or (not 0 <= input_start < input_start + input_length <= prompt_length)
):
raise ValueError("prediction_layout does not match the prompt")
kinds: list[str] = []
queries: list[int] = []
items: list[list[int]] = []
cursor = 4
for _ in range(count):
if cursor + 3 > len(layout):
raise ValueError("prediction_layout is truncated")
(kind, query, size) = layout[cursor : cursor + 3]
cursor += 3
if (
not 0 <= kind < len(FIELD_KINDS)
or not input_start + input_length <= query < prompt_length
):
raise ValueError("prediction_layout has an invalid field")
expected = LIST_SLOTS if FIELD_KINDS[kind] in LIST_KINDS else 0
slots = layout[cursor : cursor + size]
cursor += size
if (
size != expected
or len(slots) != size
or any((not query < slot < prompt_length for slot in slots))
):
raise ValueError("prediction_layout has invalid item positions")
kinds.append(FIELD_KINDS[kind])
queries.append(query + shift)
items.append([slot + shift for slot in slots])
slot_fields: list[int] = []
answer_slots: list[list[int]] = []
if cursor < len(layout):
slotted = layout[cursor]
cursor += 1
if not 0 < slotted <= count:
raise ValueError("prediction_layout has invalid answer slots")
for _ in range(slotted):
if cursor + 2 > len(layout):
raise ValueError("prediction_layout is truncated")
(index, size) = layout[cursor : cursor + 2]
cursor += 2
positions = layout[cursor : cursor + size]
cursor += size
if (
not 0 <= index < count
or (slot_fields and index <= slot_fields[-1])
or size < 1
or (len(positions) != size)
or any(
(
not queries[index] - shift < slot < prompt_length
for slot in positions
)
)
):
raise ValueError("prediction_layout has invalid answer slots")
slot_fields.append(index)
answer_slots.append([slot + shift for slot in positions])
if cursor != len(layout):
raise ValueError("prediction_layout has trailing values")
parsed = (input_start + shift, input_length, kinds, queries, items)
return (*parsed, slot_fields, answer_slots) if with_slots else parsed
def prediction_views(
kinds: list[str], queries: list[int], items: list[list[int]], device: Any = None
) -> dict[str, torch.Tensor]:
views = {"query_positions": torch.tensor(queries, dtype=torch.long, device=device)}
for group, members in VIEW_GROUPS:
views[f"{group}_rows"] = torch.tensor(
[row for (row, kind) in enumerate(kinds) if kind in members],
dtype=torch.long,
device=device,
)
for kind in LIST_KINDS:
rows = [row for (row, field_kind) in enumerate(kinds) if field_kind == kind]
views[f"{kind}_rows"] = torch.tensor(rows, dtype=torch.long, device=device)
views[f"{kind}_items"] = torch.tensor(
[slot for row in rows for slot in items[row]],
dtype=torch.long,
device=device,
).view(len(rows), LIST_SLOTS)
return views
def compact_output(output: dict[str, Any]) -> torch.Tensor:
parts = [output["null"].float().flatten()]
if "text" in output:
scores = output["text"].float().flatten(1)
parts += [
scores.argmax(dim=1).float(),
torch.softmax(scores, dim=1).amax(dim=1),
]
if "tags" in output:
tag_log = torch.log_softmax(output["tags"].float(), dim=-1)
parts.append(tag_log.argmax(dim=-1).float().flatten())
parts.append(tag_log.amax(dim=-1).sum(dim=-1))
if "trim" in output:
parts.append(output["trim"].argmax(dim=-1).float().flatten())
if "number_copy" in output:
scores = output["number_copy"].float().flatten(1)
parts += [
scores.argmax(dim=1).float(),
torch.softmax(scores, dim=1).amax(dim=1),
]
parts += [output[key].float().flatten() for key in COMPACT_KEYS if key in output]
if "slot_ids" in output:
parts += [
output["slot_ids"].float().flatten(),
output["slot_log_probs"].float().flatten(),
]
return torch.cat(parts)
def expand_output(
flat: torch.Tensor,
record: EncodedRecord,
bins: int = BINS,
estimate_lexical: bool = False,
span_trim: bool = False,
number_groups: bool = False,
number_pointer: bool = False,
) -> dict[str, Any]:
flat = flat.float().cpu()
kinds = [item.kind for item in record.fields]
rows = {
group: sum((kind in members for kind in kinds))
for (group, members) in VIEW_GROUPS
}
rows.update({kind: kinds.count(kind) for kind in LIST_KINDS})
cursor = 0
def take(*shape: int) -> torch.Tensor:
nonlocal cursor
size = math.prod(shape)
if cursor + size > flat.numel():
raise ValueError("compact output is shorter than the layout needs")
value = flat[cursor : cursor + size].view(*shape)
cursor += size
return value
output: dict[str, Any] = {"null": take(len(kinds))}
if rows["text"]:
output["text_best"] = take(rows["text"]).round().long().tolist()
output["text_probability"] = take(rows["text"]).tolist()
if rows["tags"]:
output["tags_labels"] = (
take(rows["tags"], len(record.offsets)).round().long().tolist()
)
output["tags_log_probability"] = take(rows["tags"]).tolist()
if span_trim and (rows["text"] or rows["tags"]):
output["trim_labels"] = take(len(record.offsets), 2).round().long().tolist()
if number_pointer and rows["number"]:
output["number_copy_best"] = take(rows["number"]).round().long().tolist()
output["number_copy_probability"] = take(rows["number"]).tolist()
widths = {
"number": (NUMBER_WIDTH,),
"date": (DATE_WIDTH,),
"time": (TIME_WIDTH,),
"estimate": (bins,),
"estimate_offset": (bins,),
"estimate_overflow": (2,),
"number_list_end": (LIST_SLOTS,),
"number_list": (LIST_SLOTS, NUMBER_WIDTH),
"date_list_end": (LIST_SLOTS,),
"date_list": (LIST_SLOTS, DATE_WIDTH),
"estimate_number": (NUMBER_WIDTH,),
"estimate_gate": (),
"number_groups": (NUMBER_GROUPS, GROUP_CLASSES),
"number_list_groups": (LIST_SLOTS, NUMBER_GROUPS, GROUP_CLASSES),
"number_copy_gate": (),
}
for key in COMPACT_KEYS:
if key in ("estimate_number", "estimate_gate") and (not estimate_lexical):
continue
if key in ("number_groups", "number_list_groups") and (not number_groups):
continue
if key == "number_copy_gate" and (not number_pointer):
continue
count = rows[COMPACT_GROUPS[key]]
if count:
output[key] = take(count, *widths[key])
slotted = [len(item.answer_slots) for item in record.fields if item.answer_slots]
if slotted:
output["slot_ids"] = take(len(slotted), max(slotted)).round().long()
output["slot_log_probs"] = take(len(slotted), max(slotted))
if cursor != flat.numel():
raise ValueError("compact output is longer than the layout needs")
return output
def merge_forecasts(
predictions: dict[str, dict[str, Any]], groups: dict[str, list[str]]
) -> dict[str, dict[str, Any]]:
merged = {
name: prediction
for (name, prediction) in predictions.items()
if not any((name in names for names in groups.values()))
}
for name, names in groups.items():
steps = [predictions[step] for step in names]
merged[name] = {
key: [step[key] for step in steps]
for key in ("value", "median", "p10", "p90")
}
merged[name]["steps"] = steps
return merged
from __future__ import annotations
import calendar
import datetime
import functools
import math
import os
import re
from bisect import bisect_left, bisect_right
from dataclasses import dataclass, field
from decimal import ROUND_HALF_UP, Decimal
from typing import Any, cast
import torch
from torch import nn
from torch.nn import functional
from . import schema as zs
SYSTEM_PREFIX = "<|im_start|>system\n" + zs.SYSTEM_PREAMBLE + "\nFields:\n"
EVIDENCE_SYSTEM_PREFIX = (
"<|im_start|>system\n" + zs.EVIDENCE_SYSTEM_PREAMBLE + "\nFields:\n"
)
USER_PREFIX = "<|im_end|>\n<|im_start|>user\n"
ASSISTANT_SUFFIX = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n"
ANSWER_CUE = "\nAnswer:"
class PromptLayout:
schema_header: str
query_header: str
answer_cue: str
slot_header: str
PROMPT_LAYOUTS = (
PromptLayout("", "", ANSWER_CUE, ""),
PromptLayout(
"Field definition:\n", "\nField query:", "\nValue:", "\nAnswer field:"
),
PromptLayout(
"Requested field:\n", "\nRequested field:", "\nResult:", "\nResult for field:"
),
)
PROMPT_VARIANTS = (0, 1, 2)
def prompt_layout(
template_variant: int = 0, complete_field: bool = True
) -> PromptLayout:
if (
not isinstance(template_variant, int)
or isinstance(template_variant, bool)
or template_variant not in PROMPT_VARIANTS
):
raise ValueError("template_variant must be 0, 1, or 2")
if template_variant and (not complete_field):
raise ValueError("alternative layouts require the evidence construction")
return PROMPT_LAYOUTS[template_variant]
QUERY_TOKENS = 256
MAX_SPAN = 64
TRIM_CLASSES = 8
TRIM_INIT = 2.0
MIN_INPUT_TOKENS = 16
BINS = 256
POINTER_WIDTH = 256
NUMBER_DIGITS = 12
(MIN_EXPONENT, MAX_EXPONENT) = (-12, 15)
EXPONENT_CLASSES = MAX_EXPONENT - MIN_EXPONENT + 2
YEAR_DIGITS = 4
(YEAR_LOW, YEAR_HIGH) = (1800, 2199)
YEAR_OUTSIDE = YEAR_HIGH - YEAR_LOW + 1
YEAR_CLASSES = YEAR_OUTSIDE + 1
LIST_SLOTS = zs.MAX_LIST_ITEMS
ITEM_CUES = tuple((f"\n{index}." for index in range(1, LIST_SLOTS + 1)))
(INSIDE, BELOW, ABOVE) = (0, 1, 2)
OVERFLOW_INIT = -5.0
QUERY_ORDER_RATE = 0.8
LOOP_RANK = 256
LEXICAL_DIGITS = 4
ESTIMATE_GATE_INIT = 0.0
ESTIMATE_SPREAD_LIMIT = 2.0
(EDGE_NEGATIVE, EDGE_ZERO, EDGE_POSITIVE) = (0, 1, 2)
(TAG_OUTSIDE, TAG_BEGIN, TAG_INSIDE) = (0, 1, 2)
NUMBER_WIDTH = 2 + EXPONENT_CLASSES + NUMBER_DIGITS * 10
GROUP_DIGITS = 3
GROUP_CLASSES = 10**GROUP_DIGITS
(GROUP_LOW, GROUP_HIGH) = (MIN_EXPONENT // GROUP_DIGITS, MAX_EXPONENT // GROUP_DIGITS)
NUMBER_GROUPS = GROUP_HIGH - GROUP_LOW + 1
GROUP_RANK = 256
COPY_CATEGORIES: tuple[str | None, ...] | None = ("extraction", None)
COPY_GATE_INIT = -3.0
NUMERAL_RUN = re.compile(
"(?:(?<![\\d.])\\.(?=\\d))?\\d+(?:[,.'\\u00a0\\u202f\\u2009\\u2007 \\u066b\\u066c]\\d+)*"
)
NUMERAL_SEPARATORS = ",.'\xa0\u202f\u2009\u2007 ٫٬"
NUMERAL_READINGS = (
(re.compile("\\d{1,3}(?:,\\d{3})+(?:\\.\\d+)?|\\d+(?:\\.\\d+)?|\\.\\d+"), ",", "."),
(
re.compile(