from __future__ import annotations from collections.abc import Callable from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from .api import PredictRequest from .backend import EncodedRequest from copy import deepcopy from jsonschema import Draft202012Validator, ValidationError from . import ar, identifier_scope VERSION = "identifier-codepoint-spelling-v1" PDF_VIEW_ID = "pdf-po-codepoint-spelling-focus-v1" MANY_IMAGE_VIEW_ID = "po-three-four-images-detail3-800000px-v2" MANY_IMAGE_MAX_PIXELS = 800000 INSTRUCTION = "Spell the printed purchase order reference with one space between each individual character. Preserve the actual characters; do not normalize letters and digits." def decode_wire(value: object) -> str: if not isinstance(value, str): raise ValueError("Identifier spelling wire must be a string") if any((55296 <= ord(character) <= 57343 for character in value)): raise ValueError("Identifier spelling wire contains an unpaired surrogate") if value and ( len(value) % 2 == 0 or any((character != " " for character in value[1::2])) ): raise ValueError("Invalid identifier spelling separators") return value[::2] def prepare( record: dict[str, Any], public_fields: dict[str, dict[str, Any]], query_labels: dict[str, str], *, has_media: bool, ) -> dict[str, Any]: if not has_media or not query_labels: return record (_, eligible) = identifier_scope.prepare_fields(public_fields, has_media=True) contracts = {} for name, label in query_labels.items(): if label != "purchase_order_reference" or eligible.get(name) != label: continue public = public_fields[name] spec = record["fields"][name] wire = { key: value for (key, value) in spec.items() if key not in {"minLength", "maxLength", "pattern"} } wire["instructions"] = INSTRUCTION + "\n" + spec.get("instructions", "") record["fields"][name] = wire contracts[name] = { "scalar": deepcopy(public), "wire": VERSION, "query_name": name, } if contracts: record["_meta"]["identifier_spelling"] = contracts return record def normalize_predictions( record: dict[str, Any], predictions: dict[str, dict[str, Any]] ) -> dict[str, dict[str, Any]]: result = dict(predictions) for name, contract in ( record.get("_meta", {}).get("identifier_spelling", {}).items() ): original = result.get(name) if ( original is None or not original.get("valid") or (not original.get("complete")) or (original.get("wire_decoder") == VERSION) ): continue answer = dict(original) wire = original["value"] answer.update( wire_value=wire, wire_decoder=VERSION, native_decoder=original.get("decoder"), decoder=ar.VERSION + "+" + VERSION, score_representation="native_json_codepoint_spelling_wire", ) if answer.get("span") is not None: answer["native_wire_span"] = answer["span"] if "span_method" in answer: answer["native_wire_span_method"] = answer.pop("span_method") answer["span"] = None try: decoded = None if wire is None else decode_wire(wire) Draft202012Validator(ar.schema(contract["scalar"])).validate(decoded) answer["value"] = decoded except (ValueError, TypeError, ValidationError): answer.update( value=None, valid=False, failure_reason="invalid_identifier_spelling" ) result[name] = answer return result def prepare_pdf_tasks( request: PredictRequest, encoded: EncodedRequest, encode: Callable[..., EncodedRequest], *, enabled: bool, ) -> EncodedRequest: if not enabled or not request.documents: return encoded (record, forecasts, tasks, typed, eos) = encoded active = {name for (name, _, _) in tasks} selected = { name: request.fields[name] for (name, contract) in record.get("_meta", {}) .get("identifier_spelling", {}) .items() if name in active and contract.get("wire") == VERSION } originals = record.get("_original_images", ()) pdf_pages = originals[len(request.images) :] if not selected or not pdf_pages: return encoded child_request = request.model_copy(update={"fields": selected}) (child, _, replacements, child_typed, child_eos) = encode( child_request, identifier_view=True, pdf_spelling_view=True, image_views=lambda: originals, image_view_key=PDF_VIEW_ID, ) if child_typed is not None or child_eos != eos: raise ValueError("A PDF PO spelling view must retain the scalar AR contract") by_name = {name: (name, ids, media) for (name, ids, media) in replacements} if set(by_name) != set(selected): raise ValueError("PDF PO spelling tasks differ from the selected public fields") updated = [by_name.get(task[0], task) for task in tasks] salts = dict(record.get("_ar_task_salts", {})) for name in selected: record["fields"][name] = child["fields"][name] record["_ar_constraints"][name] = child["_ar_constraints"][name] salts[name] = child["_cache_salt"] record["_meta"]["identifier_spelling_pdf_view"] = { "version": PDF_VIEW_ID, "fields": list(selected), "image_count": len(originals), "pdf_page_count": len(pdf_pages), } record.update( _ar_tasks=updated, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[] ) return (record, forecasts, updated, typed, eos) def prepare_fallback( request: PredictRequest, encoded: EncodedRequest, encode: Callable[..., EncodedRequest], ) -> EncodedRequest: (record, _, tasks, _, eos) = encoded contracts = record.get("_meta", {}).get("identifier_spelling", {}) active = {name for (name, _, _) in tasks} selected = {name: request.fields[name] for name in contracts if name in active} if not selected: return encoded child_request = request.model_copy(update={"fields": selected}) (child, _, child_tasks, child_typed, child_eos) = encode( child_request, identifier_spelling_disabled=True ) if ( child_typed is not None or child_eos != eos or {name for (name, _, _) in child_tasks} != set(selected) or child.get("_meta", {}).get("identifier_spelling") ): raise ValueError( "Identifier spelling fallback must retain the original scalar contract" ) record["_identifier_spelling_fallback"] = child return encoded def prepare_many_image_tasks( request: PredictRequest, encoded: EncodedRequest, encode: Callable[..., EncodedRequest], ) -> EncodedRequest: if request.documents or len(request.images) not in (3, 4): return encoded (record, forecasts, tasks, typed, eos) = encoded active = {name for (name, _, _) in tasks} selected = { name: request.fields[name] for (name, contract) in record.get("_meta", {}) .get("identifier_spelling", {}) .items() if name in active and contract.get("wire") == VERSION } if not selected: return encoded originals = record.get("_original_images", ()) if len(originals) != len(request.images): raise ValueError("PO detail view must retain every original image in order") (child, _, replacements, child_typed, child_eos) = encode( request.model_copy(update={"fields": selected}), identifier_view=True, po_detail_view=True, image_views=lambda: originals, image_view_key=MANY_IMAGE_VIEW_ID, ) if child_typed is not None or child_eos != eos: raise ValueError("A PO image detail view must retain the scalar AR contract") by_name = {name: (name, ids, media) for (name, ids, media) in replacements} if set(by_name) != set(selected): raise ValueError("PO image detail tasks differ from the selected public fields") updated = [by_name.get(task[0], task) for task in tasks] salts = dict(record.get("_ar_task_salts", {})) for name in selected: record["fields"][name] = child["fields"][name] record["_ar_constraints"][name] = child["_ar_constraints"][name] salts[name] = child["_cache_salt"] record["_meta"]["identifier_spelling_image_view"] = { "version": MANY_IMAGE_VIEW_ID, "fields": list(selected), "image_count": len(originals), "max_scale": 3.0, "max_pixels": MANY_IMAGE_MAX_PIXELS, } record.update( _ar_tasks=updated, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[] ) return (record, forecasts, updated, typed, eos) def fallback_record( record: dict[str, Any], predictions: dict[str, dict[str, Any]] ) -> dict[str, Any] | None: selected = { name for (name, prediction) in predictions.items() if prediction.get("failure_reason") == "invalid_identifier_spelling" } fallback = record.get("_identifier_spelling_fallback") if not selected or fallback is None: return None tasks = [task for task in fallback["_ar_tasks"] if task[0] in selected] return ( { **fallback, "_ar_tasks": tasks, "_ar_prefix": [], "_ar_group_prefixes": [], "_ar_media_groups": [], "_null_recovery": True, } if tasks else None ) def merge_fallback( predictions: dict[str, dict[str, Any]], alternatives: dict[str, dict[str, Any]], fallback: dict[str, Any], ) -> dict[str, dict[str, Any]]: result = dict(predictions) lengths = {name: len(ids) for (name, ids, _) in fallback["_ar_tasks"]} for name, alternative in alternatives.items(): original = predictions[name] result[name] = { **alternative, "identifier_spelling_fallback": True, "identifier_spelling_attempt": dict(original), "score_representation": "native_json_scalar_fallback", "tokens": original["tokens"] + alternative["tokens"], "retry_input_tokens": original.get("retry_input_tokens", 0) + lengths[name], } return result