File size: 14,052 Bytes
d82db76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 | """Table reconstruction and export helpers."""
from __future__ import annotations
import csv
import html
import io
import re
from pathlib import Path
from dataclasses import replace
from .normalization import normalize_ocr_text
from .schemas import BBox, Table, TableCell, TextLine, union_bbox
PIPE_ROW_RE = re.compile(r"^\s*\|?.+\|.+\|?\s*$")
MULTISPACE_RE = re.compile(r"\s{2,}")
LIST_CONTINUATION_RE = re.compile(r"^\s*(?:[-–—•]|\(?[A-Za-z0-9]{1,3}[\).]|[०-९]{1,3}[\).])\s+")
MAX_NORMALIZED_GRID_WIDTH = 12
def looks_like_table_line(text: str) -> bool:
stripped = text.strip()
if not stripped:
return False
if "\t" in stripped:
return True
if PIPE_ROW_RE.match(stripped):
return True
return len(MULTISPACE_RE.split(stripped)) >= 3
def split_table_row(text: str) -> list[str]:
stripped = text.strip()
if "\t" in stripped:
return [part.strip() for part in stripped.split("\t")]
if "|" in stripped:
parts = [part.strip() for part in stripped.strip("|").split("|")]
return [part for part in parts if part]
return [part.strip() for part in MULTISPACE_RE.split(stripped) if part.strip()]
def infer_table_from_lines(lines: list[TextLine], table_id: str) -> Table:
cells: list[TableCell] = []
for row_index, line in enumerate(lines):
for col_index, text in enumerate(split_table_row(line.text)):
cells.append(TableCell(row=row_index, col=col_index, text=text, bbox=line.bbox, confidence=line.confidence))
confidence_values = [line.confidence for line in lines if line.confidence is not None]
confidence = sum(confidence_values) / len(confidence_values) if confidence_values else None
return Table(
table_id=table_id,
page_index=lines[0].page_index if lines else 0,
bbox=union_bbox([line.bbox for line in lines]),
cells=cells,
confidence=confidence,
)
def table_to_grid(table: Table) -> list[list[str]]:
return cells_to_grid(table.cells)
def cells_to_grid(cells: list[TableCell]) -> list[list[str]]:
if not cells:
return []
rows = max(cell.row + cell.rowspan for cell in cells)
cols = max(cell.col + cell.colspan for cell in cells)
grid = [["" for _ in range(cols)] for _ in range(rows)]
for cell in cells:
grid[cell.row][cell.col] = cell.text
return grid
def grid_to_csv(grid: list[list[str]]) -> str:
output = io.StringIO()
writer = csv.writer(output, lineterminator="\n")
writer.writerows(grid)
return output.getvalue()
def grid_to_html(grid: list[list[str]]) -> str:
rows = []
for row in grid:
cells = "".join(f"<td>{html.escape(value)}</td>" for value in row)
rows.append(f"<tr>{cells}</tr>")
return "<table>\n" + "\n".join(rows) + "\n</table>"
def cells_to_html(cells: list[TableCell]) -> str:
if not cells:
return "<table>\n</table>"
rows = max(cell.row + cell.rowspan for cell in cells)
cols = max(cell.col + cell.colspan for cell in cells)
by_row: dict[int, dict[int, TableCell]] = {}
for cell in cells:
by_row.setdefault(cell.row, {})[cell.col] = cell
covered: set[tuple[int, int]] = set()
rendered_rows: list[str] = []
for row_index in range(rows):
rendered_cells: list[str] = []
col_index = 0
while col_index < cols:
if (row_index, col_index) in covered:
col_index += 1
continue
cell = by_row.get(row_index, {}).get(col_index)
if cell is None:
rendered_cells.append("<td></td>")
col_index += 1
continue
attrs = []
if cell.rowspan > 1:
attrs.append(f'rowspan="{cell.rowspan}"')
if cell.colspan > 1:
attrs.append(f'colspan="{cell.colspan}"')
for row_offset in range(cell.rowspan):
for col_offset in range(cell.colspan):
position = (row_index + row_offset, col_index + col_offset)
if position != (row_index, col_index):
covered.add(position)
attr_text = (" " + " ".join(attrs)) if attrs else ""
rendered_cells.append(f"<td{attr_text}>{html.escape(cell.text)}</td>")
col_index += cell.colspan
rendered_rows.append("<tr>" + "".join(rendered_cells) + "</tr>")
return "<table>\n" + "\n".join(rendered_rows) + "\n</table>"
def infer_repeated_header_colspans(table: Table, *, max_header_rows: int = 2) -> tuple[Table, int]:
"""Collapse adjacent identical header cells into colspan cells.
This is intentionally narrow: it only looks at the first few rows and only
collapses a repeated run when a following row contains distinct detail cells
beneath the run. That avoids rewriting ordinary repeated body values.
"""
if max_header_rows < 1 or not table.cells:
return table, 0
rows: dict[int, list[TableCell]] = {}
for cell in table.cells:
rows.setdefault(cell.row, []).append(cell)
changed = 0
replaced_cells: list[TableCell] = []
removed: set[int] = set()
cell_ids = {id(cell): index for index, cell in enumerate(table.cells)}
for row_index in sorted(rows):
row_cells = sorted(rows[row_index], key=lambda cell: cell.col)
if row_index >= max_header_rows or not _has_distinct_detail_row(rows, row_index):
continue
scan_index = 0
while scan_index < len(row_cells):
cell = row_cells[scan_index]
normalized = normalize_ocr_text(cell.text, collapse_spaces=True).strip()
if not normalized or cell.colspan != 1 or cell.rowspan != 1:
scan_index += 1
continue
run = [cell]
next_index = scan_index + 1
while next_index < len(row_cells):
next_cell = row_cells[next_index]
next_text = normalize_ocr_text(next_cell.text, collapse_spaces=True).strip()
if next_cell.col != run[-1].col + run[-1].colspan:
break
if next_cell.colspan != 1 or next_cell.rowspan != 1 or next_text != normalized:
break
run.append(next_cell)
next_index += 1
if len(run) < 2 or not _run_has_distinct_children(rows, row_index, run, normalized):
scan_index += 1
continue
boxes = [item.bbox for item in run if item.bbox is not None]
bbox = union_bbox(boxes) if boxes else cell.bbox
confidence_values = [item.confidence for item in run if item.confidence is not None]
confidence = sum(confidence_values) / len(confidence_values) if confidence_values else cell.confidence
replaced_cells.append(replace(cell, colspan=len(run), bbox=bbox, confidence=confidence))
removed.update(cell_ids[id(item)] for item in run)
changed += len(run) - 1
scan_index = next_index
if not changed:
return table, 0
merged_cells: list[TableCell] = []
inserted_by_position = {(cell.row, cell.col): cell for cell in replaced_cells}
for index, cell in enumerate(table.cells):
replacement = inserted_by_position.get((cell.row, cell.col))
if replacement is not None:
merged_cells.append(replacement)
continue
if index in removed:
continue
merged_cells.append(cell)
metadata = dict(table.metadata)
metadata["inferred_repeated_header_colspans"] = changed
return replace(table, cells=merged_cells, html=cells_to_html(merged_cells), metadata=metadata), changed
def normalize_table_grid_structure(table: Table) -> tuple[Table, int]:
"""Remove obvious non-content columns from a reconstructed table.
The rules are intentionally conservative and reference-free:
- drop columns that are entirely empty
- drop columns that only contain separator glyphs
- drop exact duplicate adjacent columns
"""
if any(cell.rowspan > 1 or cell.colspan > 1 for cell in table.cells):
metadata = dict(table.metadata)
metadata["normalized_table_grid_skipped_reason"] = "explicit_spans"
return replace(table, metadata=metadata), 0
grid = table_to_grid(table)
if not grid:
return table, 0
width = max(len(row) for row in grid)
if width <= 1:
return table, 0
if width > MAX_NORMALIZED_GRID_WIDTH:
return table, 0
padded = [row + [""] * (width - len(row)) for row in grid]
keep: list[int] = []
dropped = 0
previous_kept_values: list[str] | None = None
for col_index in range(width):
values = [normalize_ocr_text(row[col_index], collapse_spaces=True).strip() for row in padded]
non_empty = [value for value in values if value]
if not non_empty:
dropped += 1
continue
if all(_is_separator_cell(value) for value in non_empty):
dropped += 1
continue
if previous_kept_values is not None and values == previous_kept_values and any(values):
dropped += 1
continue
keep.append(col_index)
previous_kept_values = values
if not keep:
return table, 0
normalized_grid = [[row[col_index] for col_index in keep] for row in padded]
normalized_grid, merged_rows = _merge_sparse_bullet_continuation_rows(normalized_grid)
if dropped == 0 and merged_rows == 0:
return table, 0
cells = cells_from_grid(normalized_grid)
metadata = dict(table.metadata)
if dropped:
metadata["normalized_table_grid_dropped_columns"] = dropped
if merged_rows:
metadata["normalized_table_grid_merged_rows"] = merged_rows
normalized = replace(table, cells=cells, html=cells_to_html(cells), metadata=metadata)
return normalized, dropped + merged_rows
def _is_separator_cell(value: str) -> bool:
return bool(value) and all(char in {"-", "–", "—", "_", " "} for char in value)
def _merge_sparse_bullet_continuation_rows(grid: list[list[str]]) -> tuple[list[list[str]], int]:
if len(grid) < 2:
return grid, 0
output: list[list[str]] = []
merged = 0
for row in grid:
non_empty = [(index, normalize_ocr_text(value, collapse_spaces=True).strip()) for index, value in enumerate(row) if value.strip()]
if output and len(non_empty) == 1:
source_col, text = non_empty[0]
target_col = _continuation_target_column(output[-1], source_col=source_col, text=text)
if target_col is not None:
output[-1][target_col] = "\n".join(part for part in [output[-1][target_col].strip(), text] if part)
merged += 1
continue
if non_empty:
output.append(list(row))
elif not output:
output.append(list(row))
else:
merged += 1
return output, merged
def _continuation_target_column(previous_row: list[str], *, source_col: int, text: str) -> int | None:
if source_col != 0 or not _looks_like_list_continuation(text):
return None
candidates = [
(index, normalize_ocr_text(value, collapse_spaces=True).strip())
for index, value in enumerate(previous_row)
if index > source_col and value.strip()
]
list_candidates = [item for item in candidates if _looks_like_list_continuation(item[1])]
if list_candidates:
return list_candidates[0][0]
return None
def _looks_like_list_continuation(value: str) -> bool:
return bool(LIST_CONTINUATION_RE.match(value))
def _has_distinct_detail_row(rows: dict[int, list[TableCell]], row_index: int) -> bool:
current_cols = {cell.col for cell in rows.get(row_index, []) if cell.text.strip()}
next_cols = {cell.col for cell in rows.get(row_index + 1, []) if cell.text.strip()}
return len(next_cols) >= max(2, len(current_cols))
def _run_has_distinct_children(rows: dict[int, list[TableCell]], row_index: int, run: list[TableCell], normalized: str) -> bool:
child_by_col = {cell.col: cell for cell in rows.get(row_index + 1, [])}
child_texts: list[str] = []
for col in range(run[0].col, run[-1].col + 1):
child = child_by_col.get(col)
if child is None:
return False
child_text = normalize_ocr_text(child.text, collapse_spaces=True).strip()
if not child_text or child_text == normalized:
return False
child_texts.append(child_text)
return len(set(child_texts)) > 1
def table_to_markdown(table: Table) -> str:
grid = table_to_grid(table)
if not grid:
return ""
width = max(len(row) for row in grid)
padded = [row + [""] * (width - len(row)) for row in grid]
header = padded[0]
separator = ["---"] * width
body = padded[1:]
def render_row(row: list[str]) -> str:
return "| " + " | ".join(value.replace("\n", " ") for value in row) + " |"
return "\n".join([render_row(header), render_row(separator), *[render_row(row) for row in body]])
def write_table_files(table: Table, out_dir: Path) -> Table:
out_dir.mkdir(parents=True, exist_ok=True)
grid = table_to_grid(table)
csv_text = grid_to_csv(grid)
html_text = table.html or cells_to_html(table.cells)
csv_path = out_dir / f"{table.table_id}.csv"
html_path = out_dir / f"{table.table_id}.html"
csv_path.write_text(csv_text, encoding="utf-8")
html_path.write_text(html_text, encoding="utf-8")
table.csv_path = str(csv_path)
table.html_path = str(html_path)
table.html = html_text
return table
def cells_from_grid(grid: list[list[str]]) -> list[TableCell]:
cells: list[TableCell] = []
for row_index, row in enumerate(grid):
for col_index, text in enumerate(row):
cells.append(TableCell(row=row_index, col=col_index, text=text, bbox=BBox(0, 0, 0, 0)))
return cells
|