File size: 8,872 Bytes
f65336d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, cast

if TYPE_CHECKING:
    from .api import PredictRequest
    from numpy.typing import ArrayLike
    from .backend import EncodedRequest
import math
from dataclasses import dataclass
import numpy as np
from PIL import Image
from .schema import MAX_LIST_ITEMS


@dataclass(frozen=True)
class RowTable:
    header: tuple[int, int]
    rows: tuple[tuple[int, int], ...]

    def views(self, image: Image.Image) -> list[Image.Image]:
        result = []
        for top, bottom in self.rows:
            header = image.crop(
                (
                    0,
                    max(0, self.header[0] - 3),
                    image.width,
                    min(image.height, self.header[1] + 3),
                )
            )
            row = image.crop(
                (0, max(0, top - 3), image.width, min(image.height, bottom + 3))
            )
            view = Image.new(
                "RGB", (image.width, header.height + row.height + 12), "white"
            )
            view.paste(header, (0, 0))
            view.paste(row, (0, header.height + 12))
            result.append(view)
        return result


def _runs(values: ArrayLike) -> list[tuple[int, int]]:
    padded = np.pad(np.asarray(values, dtype=np.int8), (1, 1))
    edges = np.flatnonzero(np.diff(padded))
    return cast(
        list[tuple[int, int]], [tuple(pair) for pair in edges.reshape(-1, 2).tolist()]
    )


def detect(image: Image.Image) -> RowTable | None:
    if image.mode != "RGB" or image.width < 100 or image.height < 60:
        return None
    grey = np.asarray(image.convert("L"))
    if np.mean(grey < 245) > 0.22:
        return None
    ink = grey < 200
    bands = _runs(ink.any(axis=1))
    height: float
    candidates = []
    for index, (top, bottom) in enumerate(bands):
        height = bottom - top
        if height < 10:
            continue
        band = ink[top:bottom]
        columns = band.any(axis=0)
        separators = []
        for left, right in _runs(band.sum(axis=0) >= 0.9 * height):
            if (
                right - left > max(3, height * 0.18)
                or left < 3
                or right > image.width - 3
                or columns[left - 3 : left].any()
                or columns[right : right + 3].any()
            ):
                continue
            separators.append((left + right) / 2)
        if len(separators) >= 2:
            candidates.append((index, top, bottom, separators))
    if len(candidates) < 4:
        return None
    groups: list[list[tuple[int, int, int, list[float]]]] = []
    for candidate in candidates:
        if not groups or candidate[0] != groups[-1][-1][0] + 1:
            groups.append([])
        groups[-1].append(candidate)
    if len(groups) != 1:
        return None
    eligible = []
    for group in groups:
        if len(group) < 4 or len({len(row[3]) for row in group}) != 1:
            continue
        (header, body) = (group[0], group[1:])
        heights = np.array([bottom - top for (_, top, bottom, _) in group])
        height = float(np.median(heights))
        positions = np.array([row[3] for row in body])
        middle = np.median(positions, axis=0)
        spacing = np.diff([row[1] for row in group])
        if (
            np.max(heights) > 1.2 * np.min(heights)
            or np.max(spacing) > 1.3 * np.min(spacing)
            or np.max(np.abs(positions - middle)) > height * 1.2
            or (abs(header[3][0] - middle[0]) < height * 2)
        ):
            continue
        eligible.append(
            RowTable((header[1], header[2]), tuple(((row[1], row[2]) for row in body)))
        )
    return eligible[0] if len(eligible) == 1 else None


def prepare(
    request: PredictRequest,
    encoded: EncodedRequest,
    encode: Callable[..., EncodedRequest],
    *,
    scale: float = 2,
) -> EncodedRequest:
    (record, forecasts, tasks, typed, eos) = encoded
    annotations = record["_meta"]["extraction_contract"]["fields"]
    parents = [
        name
        for (name, annotation) in annotations.items()
        if annotation.get("image_row_fields")
    ]
    if not parents or len(request.images) != 1 or request.documents:
        return encoded
    originals = record.get("_original_images")
    image = originals[0] if originals else request.images[0].to_pil()
    table = detect(image)
    if table is None:
        return encoded
    views = table.views(image)
    salts = dict(record.get("_ar_task_salts", {}))
    for name in parents:
        spec = request.fields[name]
        if len(views) > spec.get("max_items", MAX_LIST_ITEMS):
            continue
        annotation = annotations[name]
        previous = annotation["image_row_fields"]
        replacements = []
        for index, view in enumerate(views):
            factor = min(scale, (1600000 / (view.width * view.height)) ** 0.5)
            if factor > 1:
                view = view.resize(
                    (int(view.width * factor), int(view.height * factor)),
                    Image.Resampling.BICUBIC,
                )
            child = previous[index]
            view_field = "source_row"
            row_spec = {
                "type": "object",
                "nullable": True,
                "properties": spec["items"]["properties"],
                "instructions": "Check whether the single data row below the column header matches the requested scope. Return null for an excluded row. Otherwise return the complete printed row. "
                + spec.get("instructions", "")
                + " Missing cells are null. Do not count or select a different row.",
            }
            row_request = request.model_copy(update={"fields": {view_field: row_spec}})
            (row_record, _, row_tasks, _, _) = encode(
                row_request,
                physical_row_view=True,
                image_views=lambda view=view: (view,),
                image_view_key=f"table-row-v1:{table.header}:{table.rows[index]}:{scale}",
            )
            assert len(row_tasks) == 1
            record["fields"][child] = row_record["fields"][view_field]
            record["_ar_constraints"][child] = row_record["_ar_constraints"][view_field]
            salts[child] = row_record["_cache_salt"]
            (_, ids, payload) = row_tasks[0]
            replacements.append((child, ids, payload))
        replaced = {name, *previous}
        tasks = [task for task in tasks if task[0] not in replaced] + replacements
        retained = {task[0] for task in replacements}
        for child in set(previous) - retained:
            record["fields"].pop(child)
            record["_ar_constraints"].pop(child, None)
            annotations.pop(child, None)
        annotation["geometric_row_fields"] = [task[0] for task in replacements]
        annotation.pop("image_row_fields")
        record["fields"][name].pop("_image_row_inventory", None)
    if not any(
        (annotation.get("geometric_row_fields") for annotation in annotations.values())
    ):
        return encoded
    record.update(
        _ar_tasks=tasks, _ar_task_salts=salts, _ar_prefix=[], _ar_group_prefixes=[]
    )
    return (record, forecasts, tasks, typed, eos)


def assemble(
    record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
    result = {}
    for name, annotation in record["_meta"]["extraction_contract"]["fields"].items():
        queries = annotation.get("geometric_row_fields")
        if not queries:
            continue
        rows = [predictions[query] for query in queries]
        selected = [row for row in rows if row.get("value") is not None]
        complete = all((row.get("complete", False) for row in rows))
        valid = complete and all((row.get("valid", False) for row in rows))
        answer = {
            "value": [row["value"] for row in selected],
            "complete": complete,
            "valid": valid,
            "confidence": 0.0,
            "confidences": [row.get("confidence", 0.0) for row in selected],
            "confidence_method": "ar_physical_image_row_sequence_probability",
            "p_null": math.prod((row["p_null"] for row in rows)),
            "p_nulls": [row["p_null"] for row in selected],
            "p_null_method": "product_physical_image_row_exclusion_probabilities",
            "spans": [None] * len(selected),
        }
        if valid and all(("log_probability" in row for row in rows)):
            probability = sum((row["log_probability"] for row in rows))
            if math.isfinite(probability):
                answer.update(
                    log_probability=probability,
                    confidence=math.exp(min(0.0, probability)),
                )
        result[name] = answer
    return result