clairic / clairic_runtime /grounding.py
areneau's picture
Add complete Clairic Transformers package
f65336d verified
Raw History Blame Contribute Delete
16 kB
from __future__ import annotations
import json
import re
import unicodedata
from decimal import Decimal, InvalidOperation, localcontext
from typing import Any
from . import ar
MAX_GRAMMAR_CHARACTERS = 4096
_STATE_ONLY = re.compile(
"\\b(?:from (?:the )?(?:accompanying (?:state )?|state )text only|supplied in the accompanying state text)\\b",
re.IGNORECASE,
)
_NULL_ABSENCE_GUARD = re.compile(
"(?:^|(?<=[.;\\n]))\\s*return\\s+null\\s+(?:only\\s+)?if\\s+(?:not\\s+(?:explicitly\\s+)?printed|absent|missing)\\s*(?:[.;]|$)",
re.IGNORECASE,
)
_EXCLUDED_SOURCE = re.compile(
"\\b(?:not|ignore|exclude|excluding)\\b[^.;:\\n]*$", re.IGNORECASE
)
_DOCUMENT = "(?:(?:this|the)(?:\\s+(?:supplied|provided))?|supplied|provided)\\s+(?:note|document|text)"
_DOCUMENT_ONLY = re.compile(
f"\\b(?:use\\s+(?:only\\s+{_DOCUMENT}|{_DOCUMENT}\\s+only)|from\\s+{_DOCUMENT}\\s+only)\\b",
re.IGNORECASE,
)
_DOCUMENT_EXCLUDED = re.compile(
"\\b(?:not|never|don't|ignore|exclude|excluding)\\b[^.;:\\n]*$", re.IGNORECASE
)
_DERIVED_NUMBER = re.compile(
"\\b(?:calculat\\w*|comput\\w*|deriv\\w*|convert\\w*|conversion\\w*|count\\w*|sum\\w*|averag\\w*|mean|median|difference|ratio|growth|product|multipli\\w*|multiply|subtract\\w*|minus|divid\\w*|division|quotient|remainder|add|adding|addition|plus|times|percent\\w*|proportion|fraction)\\b|\\b(?:how many|how much|total number of|number of (?:line )?(?:items|rows|entries|records))\\b|[+*/\\u00d7\\u00f7=]|\\s[-\\u2212]\\s",
re.IGNORECASE,
)
_CURRENCY_CODES = frozenset(
{
"USD",
"EUR",
"GBP",
"JPY",
"CNY",
"INR",
"AUD",
"CAD",
"CHF",
"HKD",
"NZD",
"SGD",
"SEK",
"NOK",
"DKK",
"KRW",
"BRL",
"MXN",
"ZAR",
}
)
_EXACT_TEXT = re.compile(
"\\b(?:verbatim|raw|literal\\w*|print\\w*|exact\\w*|characters?|digits?|whitespace|spacing|newlines?)\\b|\\b(?:line\\s+breaks?|white\\s+space|do\\s+not\\s+normaliz\\w*)\\b",
re.IGNORECASE,
)
def allows_whitespace_normalization(spec: dict[str, Any]) -> bool:
return (
spec.get("type") == "text"
and (not spec.get("knowledge"))
and ("source_only" not in spec)
and (not spec.get("_physical_row_scope"))
and (
not any(
(key in spec for key in ("enum", "pattern", "minLength", "maxLength"))
)
)
and (not _EXACT_TEXT.search(spec.get("instructions", "")))
)
def collapse_whitespace(source: str) -> tuple[str, list[tuple[int, int]]]:
(characters, offsets) = ([], [])
for match in re.finditer("\\s+|\\S", source):
characters.append(" " if match[0].isspace() else match[0])
offsets.append(match.span())
return ("".join(characters), offsets)
def state_only(spec: dict[str, Any]) -> bool:
instructions = spec.get("instructions", "")
scope = _NULL_ABSENCE_GUARD.sub("", instructions)
return (
spec["type"] in ("text", "number")
and (not spec.get("knowledge"))
and (not re.search("\\b(?:if|unless|compare|combine)\\b", scope, re.IGNORECASE))
and any(
(
not _EXCLUDED_SOURCE.search(instructions[: match.start()])
for match in _STATE_ONLY.finditer(instructions)
)
)
)
def explicit_media_scope(spec: dict[str, Any]) -> bool:
return (
spec.get("type") in ("text", "number")
and (not spec.get("knowledge"))
and bool(
re.search(
"\\b(?:page|image)\\s+\\d+\\b",
spec.get("instructions", ""),
re.IGNORECASE,
)
)
)
def document_only(spec: dict[str, Any]) -> bool:
instructions = spec.get("instructions", "")
scope = _NULL_ABSENCE_GUARD.sub("", instructions)
return (
spec["type"] in ("text", "number")
and (not spec.get("knowledge"))
and (not re.search("\\b(?:if|unless|compare|combine)\\b", scope, re.IGNORECASE))
and (not (spec["type"] == "number" and _DERIVED_NUMBER.search(scope)))
and any(
(
not _DOCUMENT_EXCLUDED.search(instructions[: match.start()])
for match in _DOCUMENT_ONLY.finditer(instructions)
)
)
)
def applies(spec: dict[str, Any], media: object | None) -> bool:
return (
spec["type"] in ("text", "number")
and media is None
and (not spec.get("knowledge", False))
and (
spec["type"] == "text"
or spec.get("source_only", False)
or state_only(spec)
or document_only(spec)
)
)
def _document_number_span(text: str, span: tuple[int, int]) -> bool:
(start, _) = span
left = start
while left and (text[left - 1].isalnum() or text[left - 1] in "_./-"):
left -= 1
prefix = text[left:start].lstrip("_./-")
return not prefix or not prefix[0].isalpha() or prefix.upper() in _CURRENCY_CODES
def _text_states(source: str, prefix: str) -> list[str]:
transitions: list[dict[str, int]]
(transitions, links, lengths) = ([{}], [-1], [0])
last = 0
for char in source:
current = len(transitions)
transitions.append({})
lengths.append(lengths[last] + 1)
links.append(0)
parent = last
while parent >= 0 and char not in transitions[parent]:
transitions[parent][char] = current
parent = links[parent]
if parent >= 0:
target = transitions[parent][char]
if lengths[parent] + 1 == lengths[target]:
links[current] = target
else:
clone = len(transitions)
transitions.append(dict(transitions[target]))
lengths.append(lengths[parent] + 1)
links.append(links[target])
while parent >= 0 and transitions[parent].get(char) == target:
transitions[parent][char] = clone
parent = links[parent]
links[target] = links[current] = clone
last = current
quote = json.dumps('"')
rules = []
for index, edges in enumerate(transitions):
choices = [quote] if index else []
for char, target in edges.items():
escaped = json.dumps(char, ensure_ascii=False)[1:-1]
choices.append(
json.dumps(escaped, ensure_ascii=False) + f" {prefix}{target}"
)
rules.append(f"{prefix}{index} ::= " + " | ".join(choices))
return rules
def text_grammar(
source: str, *, nullable: bool = True, normalized_source: str | None = None
) -> str:
if not source:
if not nullable:
raise ValueError("Empty source permits no non-null text value")
return 'root ::= "null"'
quote = json.dumps('"')
starts = ['"null"'] if nullable else []
starts.append(quote + " s0")
states = _text_states(source, "s")
if normalized_source and normalized_source != source:
starts.append(quote + " n0")
states.extend(_text_states(normalized_source, "n"))
return "\n".join(["root ::= " + " | ".join(starts), *states])
def numbers(source: str) -> dict[Decimal, list[tuple[int, int]]]:
from .model import NUMERAL_RUN, numeral_convention, numeral_values
convention = numeral_convention(source)
result: dict[Decimal, list[tuple[int, int]]] = {}
consumed_end = 0
for run in NUMERAL_RUN.finditer(source):
if run.start() < consumed_end:
continue
readings = [(run.group(), run.start(), run.end())]
if not numeral_values(run.group(), convention):
readings = [
(m.group(), run.start() + m.start(), run.start() + m.end())
for m in re.finditer("[^, \\u00a0\\u202f\\u2009\\u2007]+", run.group())
]
for printed, left, end in readings:
values = numeral_values(printed, convention)
if not values:
continue
exponent = re.match("[eE]([-+]?\\d+)", source[end:])
if exponent:
consumed_end = end + len(exponent[0])
if len(exponent[1]) > 4 or abs(int(exponent[1])) > 308:
continue
with localcontext() as context:
context.prec = max(context.prec, len(printed) + 310)
values = [
value * Decimal(10) ** int(exponent[1]) for value in values
]
end += len(exponent[0])
start = left
while start > max(0, left - 32) and (
source[start - 1] in " \t-+−"
or unicodedata.category(source[start - 1]) == "Sc"
):
start -= 1
signed = source[start:left].strip()
negative = any((c in signed for c in "-−")) and (
start == 0 or not source[start - 1].isalnum()
)
left = start + len(source[start:left]) - len(source[start:left].lstrip())
before = source[max(0, left - 32) : left]
after = source[end : end + 64]
(prefix, suffix) = (before.rstrip(), after.lstrip())
accounting = (
any(
(
c in ".," or unicodedata.category(c) == "Sc"
for c in printed + signed
)
)
or not suffix[1:].split("\n")[0].strip()
)
if prefix.endswith("(") and suffix.startswith(")") and accounting:
negative = True
left -= len(before) - len(prefix) + 1
end += len(after) - len(suffix) + 1
for value in values:
if negative:
value = value.copy_negate()
numeric = (
int(value) if value == value.to_integral_value() else float(value)
)
if Decimal(str(numeric)) != value:
continue
result.setdefault(value, []).append((left, end))
return result
class Source:
def __init__(self, text: str, *, normalize_whitespace: bool = False) -> None:
self.text = text
self.numerals = numbers(text)
self.normalize_whitespace = normalize_whitespace
self._normalized_source: tuple[str, list[tuple[int, int]]] | None = None
self._grammars: dict[tuple[bool, bool], str] = {}
self._constraints: dict[tuple[str, bool, bool], dict[str, str]] = {}
def whitespace_normalization(self, spec: dict[str, Any]) -> bool:
return self.normalize_whitespace and allows_whitespace_normalization(spec)
def _whitespace_source(self) -> tuple[str, list[tuple[int, int]]]:
if self._normalized_source is None:
self._normalized_source = collapse_whitespace(self.text)
return self._normalized_source
def normalized_span(self, value: object) -> tuple[bool, list[int] | None]:
if not isinstance(value, str) or not value:
return (False, None)
matches = set()
start = self.text.find(value)
while start >= 0:
matches.add((start, start + len(value)))
start = self.text.find(value, start + 1)
(normalized, offsets) = self._whitespace_source()
start = normalized.find(value)
while start >= 0:
matches.add((offsets[start][0], offsets[start + len(value) - 1][1]))
start = normalized.find(value, start + 1)
return (bool(matches), list(next(iter(matches))) if len(matches) == 1 else None)
def span(
self, value: object, *, spec: dict[str, Any] | None = None
) -> tuple[bool, list[int] | None]:
if isinstance(value, str):
if spec is not None and self.whitespace_normalization(spec):
return self.normalized_span(value)
if not value:
return (False, None)
start = self.text.find(value)
if start < 0:
return (False, None)
unique = bool(value) and self.text.find(value, start + 1) < 0
return (True, [start, start + len(value)] if unique else None)
try:
number = Decimal(str(value))
except InvalidOperation:
return (False, None)
if not number.is_finite():
return (False, None)
matches = self.numerals.get(number, [])
return (bool(matches), list(matches[0]) if len(matches) == 1 else None)
def constraint(self, spec: dict[str, Any]) -> dict[str, str]:
document_numbers = (
spec["type"] == "number"
and document_only(spec)
and (not spec.get("source_only"))
and (not state_only(spec))
)
key = (
json.dumps(ar.schema(spec), ensure_ascii=False),
bool(document_numbers),
self.whitespace_normalization(spec),
)
if key not in self._constraints:
self._constraints[key] = self._constraint(
spec, document_numbers=document_numbers
)
return self._constraints[key]
def _constraint(
self, spec: dict[str, Any], *, document_numbers: bool = False
) -> dict[str, str]:
from jsonschema import Draft202012Validator
schema = ar.schema(spec)
if spec["type"] == "number" or "enum" in schema:
numerals = self.numerals
if document_numbers:
numerals = {
value: spans
for (value, spans) in numerals.items()
if any((_document_number_span(self.text, span) for span in spans))
}
candidates = schema.get("enum", list(numerals))
validator = Draft202012Validator(schema)
values = []
for candidate in candidates:
value = candidate
if isinstance(value, Decimal):
value = (
int(value)
if value == value.to_integral_value()
else float(value)
)
if (
value is not None
and validator.is_valid(value)
and (not document_numbers or Decimal(str(value)) in numerals)
and self.span(value)[0]
):
values.append(value)
if validator.is_valid(None):
values.append(None)
if not values:
raise ValueError(
"Field constraints permit no literal source value or null"
)
if spec["type"] == "number":
literals = [
"null" if value is None else str(Decimal(str(value)))
for value in values
]
return {
"ebnf": "root ::= "
+ " | ".join((json.dumps(value) for value in literals))
}
return {
"json_schema": json.dumps(
{**schema, "enum": values}, ensure_ascii=False
)
}
if len(self.text) > MAX_GRAMMAR_CHARACTERS or any(
(key in schema for key in ("minLength", "maxLength", "pattern"))
):
return {"json_schema": json.dumps(schema, ensure_ascii=False)}
nullable = spec.get("nullable", True)
normalize = self.whitespace_normalization(spec)
key = (nullable, normalize)
if key not in self._grammars:
normalized = self._whitespace_source()[0] if normalize else None
self._grammars[key] = text_grammar(
self.text, nullable=nullable, normalized_source=normalized
)
return {"ebnf": self._grammars[key]}