clairic-flash / clairic_runtime /image_ragged_rows.py
areneau's picture
Add complete Clairic Transformers package
dc7d8aa verified
Raw History Blame Contribute Delete
13.7 kB
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .api import PredictRequest
from .backend import EncodedRequest
import hashlib
import math
import re
from copy import deepcopy
from dataclasses import dataclass
from itertools import pairwise
import numpy as np
from PIL import Image
from . import image_table_rows, table_rows
from .schema import MAX_LIST_ITEMS
VIEW_ID = "ragged-table-v1"
TARGET_GATE = "target_is_billed_item"
TARGET_ROW = "target_row"
_BILLED_SCOPE = re.compile(
"\\b(?:line[ _-]?items?|billed\\s+(?:products?|services?|items?|goods))\\b",
re.IGNORECASE,
)
_FINANCIAL_SCOPE = re.compile(
"\\b(?:tax(?:es)?|fees?|surcharges?|payments?)\\b", re.IGNORECASE
)
TARGET_INSTRUCTION = "Image 1 is the complete source document. Image 2 isolates one physical row from image 1. Classify and read only the target row in image 2. Use image 1 to identify the table column labels. A column header is not a billed item. Do not select a neighboring row or count row positions. "
@dataclass(frozen=True)
class RaggedRows:
rows: tuple[tuple[int, int], ...]
def views(self, image: Image.Image) -> tuple[Image.Image, ...]:
return tuple(
(
image.crop(
(0, max(0, top - 3), image.width, min(image.height, bottom + 3))
)
for (top, bottom) in self.rows
)
)
def detect(image: Image.Image) -> RaggedRows | None:
if image_table_rows.detect(image) is not None:
return None
if image.mode != "RGB" or image.width < 100 or image.height < 60:
return None
grey = np.asarray(image.convert("L"))
if np.mean(grey < 245) > 0.22:
return None
ink = grey < 200
candidates = []
for index, (top, bottom) in enumerate(image_table_rows._runs(ink.any(axis=1))):
height = bottom - top
if height < 10:
continue
band = ink[top:bottom]
columns = band.any(axis=0)
separators = []
for left, right in image_table_rows._runs(band.sum(axis=0) >= 0.9 * height):
if (
right - left > max(3, height * 0.18)
or left < 3
or right > image.width - 3
or columns[left - 3 : left].any()
or columns[right : right + 3].any()
):
continue
separators.append((left, right))
if len(separators) >= 2:
candidates.append((index, top, bottom, separators, columns))
if len(candidates) < 2:
return None
if any((right[0] != left[0] + 1 for (left, right) in pairwise(candidates))):
return None
if len({len(row[3]) for row in candidates}) != 1:
return None
if any((row[4][row[3][-1][1] + 3 :].any() for row in candidates)):
return None
heights = np.array([bottom - top for (_, top, bottom, _, _) in candidates])
spacing = np.diff([row[1] for row in candidates])
if np.max(heights) > 1.2 * np.min(heights) or np.max(spacing) > 1.3 * np.min(
spacing
):
return None
return RaggedRows(tuple(((row[1], row[2]) for row in candidates)))
def _eligible(spec: dict[str, Any]) -> bool:
item = spec.get("items")
properties = item.get("properties", {}) if isinstance(item, dict) else {}
(scope, *exclusions) = re.split(
"\\bExclude\\b", spec.get("instructions", ""), maxsplit=1, flags=re.IGNORECASE
)
if exclusions:
if re.search(
"\\b(?:not|never|\\w+n['’]t)(?:\\s+\\w+){0,2}\\s*$", scope, re.IGNORECASE
):
return False
(excluded, *later) = re.split("[.;\\n]", exclusions[0], maxsplit=1)
if (
later
and _FINANCIAL_SCOPE.search(later[0])
or (
_FINANCIAL_SCOPE.search(excluded)
and re.search(
"\\b(?:include|including|keep|retain|except|unless|not|no|none)\\b",
excluded,
re.IGNORECASE,
)
)
):
return False
return (
spec.get("type") == "list"
and isinstance(item, dict)
and (item.get("type") == "object")
and (not table_rows.join_wrapped_descriptions(spec.get("instructions", "")))
and (not spec.get("knowledge"))
and (len(properties) >= 3)
and bool(_BILLED_SCOPE.search(scope))
and (not _FINANCIAL_SCOPE.search(scope))
and all(
(
child.get("type") in ("text", "number") and (not child.get("knowledge"))
for child in properties.values()
)
)
)
def row_fields(spec: dict[str, Any]) -> dict[str, dict[str, Any]]:
wrapped = table_rows.join_wrapped_descriptions(spec.get("instructions", ""))
membership = (
"A logical item can span several printed bands. Include it exactly once, at its first description band. Return false for a later continuation band belonging to that preceding item, even when its numeric cells are printed on the continuation band. Distinct separate source rows remain separate even if they repeat. "
if wrapped
else ""
)
readout = (
"For the target item's first band, use the full document to read its complete wrapped description and cells from its continuation bands. Never borrow from a neighboring logical item. "
if wrapped
else ""
)
return {
TARGET_GATE: {
"type": "boolean",
"nullable": False,
"instructions": TARGET_INSTRUCTION
+ membership
+ "Is the target row a billed product or service row? Return false for the column header, totals, tax, fees, or payment rows. "
+ spec.get("instructions", ""),
},
TARGET_ROW: {
"type": "object",
"nullable": False,
"properties": deepcopy(spec["items"]["properties"]),
"instructions": TARGET_INSTRUCTION
+ readout
+ "Read the complete printed target row as an object. Whether it belongs in the requested list is decided separately. Always return an object with every requested cell. Missing cells and absent columns are null. Do not relabel an Amount column as Unit price. Never compute missing values.",
},
}
def prepare(
request: PredictRequest,
encoded: EncodedRequest,
encode: Callable[..., EncodedRequest],
*,
scale: float = 2,
) -> EncodedRequest:
(record, forecasts, tasks, typed, eos) = encoded
annotations = record["_meta"]["extraction_contract"]["fields"]
parents = [
name
for (name, spec) in request.fields.items()
if _eligible(spec) and annotations.get(name, {}).get("image_row_fields")
]
if not parents or len(request.images) != 1 or request.documents:
return encoded
originals = record.get("_original_images")
image = originals[0] if originals else request.images[0].to_pil()
plan = detect(image)
if plan is None:
return encoded
views = []
for view in plan.views(image):
factor = min(scale, (1600000 / (view.width * view.height)) ** 0.5)
if factor > 1:
view = view.resize(
(int(view.width * factor), int(view.height * factor)),
Image.Resampling.BICUBIC,
)
views.append(view)
salts = dict(record.get("_ar_task_salts", {}))
gates = list(record.get("_source_row_gates", []))
changed = False
for name in parents:
(spec, annotation) = (request.fields[name], annotations[name])
previous = annotation["image_row_fields"]
maximum = spec.get("max_items", MAX_LIST_ITEMS)
if len(views) > maximum + 1 or len(views) > len(previous):
continue
(replacements, pairs) = ([], [])
for index, (view, band) in enumerate(zip(views, plan.rows, strict=True)):
child = previous[index]
digest = hashlib.sha256(child.encode()).hexdigest()[:16]
gate = "ragged_gate_" + digest
while gate in record["fields"]:
gate += "_"
row_request = request.model_copy(update={"fields": row_fields(spec)})
(row_record, _, row_tasks, _, _) = encode(
row_request,
physical_row_view=True,
image_views=lambda view=view: (image, view),
image_view_key=f"{VIEW_ID}:{band}:scale{scale}",
)
assert len(row_tasks) == 2
names = {TARGET_GATE: gate, TARGET_ROW: child}
for source, ids, payload in row_tasks:
target = names[source]
record["fields"][target] = row_record["fields"][source]
record["_ar_constraints"][target] = row_record["_ar_constraints"][
source
]
salts[target] = row_record["_cache_salt"]
annotations[target] = {"parent": name}
replacements.append((target, ids, payload))
gates.append(gate)
pairs.append((gate, child))
tasks = [
task for task in tasks if task[0] not in {name, *previous}
] + replacements
retained = {copy for (_, copy) in pairs}
for child in set(previous) - retained:
record["fields"].pop(child, None)
record["_ar_constraints"].pop(child, None)
annotations.pop(child, None)
salts.pop(child, None)
annotation.pop("image_row_fields")
annotation["ragged_row_fields"] = pairs
annotation["ragged_row_max_items"] = maximum
record["fields"][name].pop("_image_row_inventory", None)
changed = True
if not changed:
return encoded
record.update(
_ar_tasks=tasks,
_ar_task_salts=salts,
_source_row_gates=gates,
_ar_prefix=[],
_ar_group_prefixes=[],
)
record.pop("_ar_media_groups", None)
return (record, forecasts, tasks, typed, eos)
def _score(value: object, *, probability: bool = False) -> bool:
return (
isinstance(value, (int, float))
and (not isinstance(value, bool))
and math.isfinite(value)
and (0 <= value <= 1 if probability else value <= 0)
)
def assemble(
record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
result = {}
for name, annotation in record["_meta"]["extraction_contract"]["fields"].items():
pairs = annotation.get("ragged_row_fields")
if not pairs:
continue
rows = [(predictions[gate], predictions[copy]) for (gate, copy) in pairs]
selected = [(gate, copy) for (gate, copy) in rows if gate.get("value") is True]
complete = all((gate.get("complete", False) for (gate, _) in rows)) and all(
(copy.get("complete", False) for (_, copy) in selected)
)
exclusion_scored = all(
(_score(gate.get("p_excluded"), probability=True) for (gate, _) in rows)
)
relevant = [gate for (gate, _) in rows] + [copy for (_, copy) in selected]
scored = exclusion_scored and all(
(_score(answer.get("log_probability")) for answer in relevant)
)
probability = (
sum((answer["log_probability"] for answer in relevant)) if scored else None
)
if probability is not None and (not math.isfinite(probability)):
(scored, probability) = (False, None)
valid = (
complete
and scored
and all(
(
gate.get("valid", False) and type(gate.get("value")) is bool
for (gate, _) in rows
)
)
and all(
(
copy.get("valid", False) and isinstance(copy.get("value"), dict)
for (_, copy) in selected
)
)
)
within_limit = len(selected) <= annotation["ragged_row_max_items"]
valid &= within_limit
answer = {
"value": [copy["value"] for (_, copy) in selected],
"valid": valid,
"complete": complete,
"confidence": math.exp(probability)
if valid and probability is not None
else 0.0,
"confidence_method": "ar_physical_image_row_eligibility_and_copy_probability",
"confidences": [
math.exp(gate["log_probability"] + copy["log_probability"])
if scored
else 0.0
for (gate, copy) in selected
],
"p_null": math.prod((gate["p_excluded"] for (gate, _) in rows))
if exclusion_scored
else 0.0,
"p_null_method": "product_physical_image_row_exclusion_probabilities"
if exclusion_scored
else "unavailable_physical_image_row_exclusion_probability",
"p_nulls": [
gate["p_excluded"] if exclusion_scored else 0.0
for (gate, _) in selected
],
"spans": [None] * len(selected),
}
if probability is not None:
answer["log_probability"] = probability
if not valid:
answer["failure_reason"] = (
"ragged_row_output_exceeds_max_items"
if not within_limit
else "ragged_row_unscored"
if not scored
else "ragged_row_read_incomplete"
)
result[name] = answer
return result