prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
12.7 kB
"""Convert OCR lines into blocks, tables, figures, and reading order."""
from __future__ import annotations
import re
from .engines import EngineOutput
from .metrics import bbox_iou
from .normalization import normalize_ocr_text
from .schemas import Block, Document, Figure, Page, Table, TextLine, union_bbox
from .tables import infer_table_from_lines, looks_like_table_line
FIGURE_MARKER_RE = re.compile(r"^\s*(?:\[(?:figure|chart|image|चित्र)[^\]]*\]|(?:figure|fig\.?|chart|चित्र)\s*[::]?)", re.IGNORECASE)
CAPTION_RE = re.compile(r"^\s*(?:caption|figure|fig\.?|चित्र|तालिका)\s*[\s\d०-९IVXivx.-]*[::]", re.IGNORECASE)
FRONT_MATTER_STRUCTURAL_ROLES = {"page_marker"}
FRONT_MATTER_DIGIT_PREFIX_RE = re.compile(r"^\s*[०-९]\s+\S")
def sort_lines_reading_order(lines: list[TextLine]) -> list[TextLine]:
if not lines:
return []
heights = [line.bbox.h for line in lines if line.bbox.h > 0]
line_height = sorted(heights)[len(heights) // 2] if heights else 20
bucket_size = max(8, line_height * 0.75)
return sorted(lines, key=lambda line: (round(line.bbox.y / bucket_size), line.bbox.x, line.bbox.y))
def build_document(source_path: str, output: EngineOutput) -> Document:
pages: list[Page] = []
tables = list(output.tables)
figures = list(output.figures)
preserve_output_line_order = bool(output.metadata.get("preserve_line_order"))
for raw_page in output.pages:
preserve_page_line_order = preserve_output_line_order or bool((raw_page.metadata or {}).get("preserve_line_order"))
if preserve_page_line_order:
sorted_lines = list(raw_page.text_lines)
else:
sorted_lines = sort_lines_reading_order(raw_page.text_lines)
for index, line in enumerate(sorted_lines):
if line.line_id is None:
line.line_id = f"p{raw_page.page_index}-l{index}"
if preserve_page_line_order:
pages.append(
Page(
page_index=raw_page.page_index,
width=raw_page.width,
height=raw_page.height,
text_lines=sorted_lines,
blocks=_preserved_line_order_blocks(raw_page.page_index, sorted_lines),
metadata=raw_page.metadata,
)
)
continue
blocks: list[Block] = []
represented_figure_ids: set[str] = set()
order = 0
index = 0
while index < len(sorted_lines):
line = sorted_lines[index]
structural_role = _structural_role(line)
if structural_role is not None:
text = normalize_ocr_text(line.text).strip()
if text:
blocks.append(
Block(
block_id=f"p{raw_page.page_index}-b{order}",
block_type="text",
page_index=raw_page.page_index,
bbox=line.bbox,
order=order,
text=text,
confidence=line.confidence,
line_ids=[line.line_id or ""],
metadata={"structural_role": structural_role},
)
)
order += 1
index += 1
continue
if looks_like_figure_marker(line.text):
marker_line = line
caption_line: TextLine | None = None
index += 1
if index < len(sorted_lines) and _looks_like_caption_after_marker(marker_line, sorted_lines[index]):
caption_line = sorted_lines[index]
index += 1
figure_lines = [marker_line] + ([caption_line] if caption_line is not None else [])
figure_id = f"p{raw_page.page_index}-f{len(figures)}"
caption = normalize_ocr_text(caption_line.text).strip() if caption_line is not None else None
summary = normalize_ocr_text(marker_line.text).strip()
figure = Figure(
figure_id=figure_id,
page_index=raw_page.page_index,
bbox=union_bbox([item.bbox for item in figure_lines]),
caption=caption,
summary=summary,
confidence=_average_confidence(figure_lines),
metadata={"line_ids": [item.line_id or "" for item in figure_lines]},
)
figures.append(figure)
represented_figure_ids.add(figure_id)
blocks.append(
Block(
block_id=f"p{raw_page.page_index}-b{order}",
block_type="figure",
page_index=raw_page.page_index,
bbox=figure.bbox,
order=order,
figure_id=figure_id,
confidence=figure.confidence,
line_ids=[item.line_id or "" for item in figure_lines],
text=caption or summary,
)
)
order += 1
continue
if looks_like_table_line(line.text):
table_lines = [line]
index += 1
while index < len(sorted_lines) and looks_like_table_line(sorted_lines[index].text):
table_lines.append(sorted_lines[index])
index += 1
table_id = f"p{raw_page.page_index}-t{len(tables)}"
table = infer_table_from_lines(table_lines, table_id)
existing_table = _find_overlapping_table(tables, table)
represented_table = existing_table or table
if existing_table is None:
tables.append(table)
blocks.append(
Block(
block_id=f"p{raw_page.page_index}-b{order}",
block_type="table",
page_index=raw_page.page_index,
bbox=represented_table.bbox,
order=order,
table_id=represented_table.table_id,
confidence=represented_table.confidence,
line_ids=[line.line_id or "" for line in table_lines],
)
)
order += 1
continue
text_lines = [line]
index += 1
while (
index < len(sorted_lines)
and not looks_like_table_line(sorted_lines[index].text)
and not looks_like_figure_marker(sorted_lines[index].text)
and _structural_role(sorted_lines[index]) is None
):
previous = text_lines[-1]
current = sorted_lines[index]
vertical_gap = current.bbox.y - previous.bbox.bottom
if vertical_gap > max(previous.bbox.h, 24) * 1.5:
break
text_lines.append(current)
index += 1
text = "\n".join(normalize_ocr_text(item.text).strip() for item in text_lines if item.text.strip())
if not text.strip():
continue
confidence = _average_confidence(text_lines)
block_type = "title" if order == 0 and len(text) <= 120 and len(text_lines) == 1 else "text"
blocks.append(
Block(
block_id=f"p{raw_page.page_index}-b{order}",
block_type=block_type,
page_index=raw_page.page_index,
bbox=union_bbox([item.bbox for item in text_lines]),
order=order,
text=text,
confidence=confidence,
line_ids=[item.line_id or "" for item in text_lines],
)
)
order += 1
for figure in [item for item in figures if item.page_index == raw_page.page_index and item.figure_id not in represented_figure_ids]:
blocks.append(
Block(
block_id=f"p{raw_page.page_index}-b{order}",
block_type="figure",
page_index=raw_page.page_index,
bbox=figure.bbox,
order=order,
figure_id=figure.figure_id,
confidence=figure.confidence,
text=figure.caption or figure.summary or "",
)
)
order += 1
blocks = sorted(blocks, key=lambda block: _block_sort_key(raw_page.height, block))
for order_index, block in enumerate(blocks):
block.order = order_index
pages.append(
Page(
page_index=raw_page.page_index,
width=raw_page.width,
height=raw_page.height,
text_lines=sorted_lines,
blocks=blocks,
metadata=raw_page.metadata,
)
)
return Document(source_path=source_path, pages=pages, tables=tables, figures=figures, metadata=dict(output.metadata))
def _preserved_line_order_blocks(page_index: int, lines: list[TextLine]) -> list[Block]:
blocks: list[Block] = []
for order, line in enumerate(lines):
metadata: dict[str, str] | None = None
structural_role = _structural_role(line)
if structural_role is not None:
metadata = {"structural_role": structural_role}
text = normalize_ocr_text(line.text)
blocks.append(
Block(
block_id=f"p{page_index}-b{order}",
block_type="text",
page_index=page_index,
bbox=line.bbox,
order=order,
text=text,
confidence=line.confidence,
line_ids=[line.line_id or ""],
metadata=metadata,
)
)
return blocks
def _structural_role(line: TextLine) -> str | None:
role = (line.metadata or {}).get("structural_role")
if not isinstance(role, str):
return None
normalized = role.strip()
if not normalized or normalized == "text":
return None
return normalized
def _block_structural_role(block: Block) -> str | None:
role = (block.metadata or {}).get("structural_role")
if not isinstance(role, str):
return None
normalized = role.strip()
return normalized or None
def _is_bottom_digit_prefixed_block(page_height: float | int | None, block: Block) -> bool:
if page_height is None or page_height <= 0:
return False
if block.bbox.y < float(page_height) * 0.80:
return False
return bool(FRONT_MATTER_DIGIT_PREFIX_RE.match(normalize_ocr_text(block.text or "")))
def _block_sort_key(page_height: float | int | None, block: Block) -> tuple[int, float, float, int]:
role = _block_structural_role(block)
if role in FRONT_MATTER_STRUCTURAL_ROLES:
front_matter_rank = 0
elif _is_bottom_digit_prefixed_block(page_height, block):
front_matter_rank = 1
else:
front_matter_rank = 2
return (front_matter_rank, block.bbox.y, block.bbox.x, block.order)
def looks_like_figure_marker(text: str) -> bool:
return bool(FIGURE_MARKER_RE.match(text.strip()))
def looks_like_caption(text: str) -> bool:
return bool(CAPTION_RE.match(text.strip()))
def _looks_like_caption_after_marker(marker_line: TextLine, candidate: TextLine) -> bool:
if looks_like_caption(candidate.text):
return True
if looks_like_figure_marker(candidate.text) or looks_like_table_line(candidate.text):
return False
if not candidate.text.strip():
return False
vertical_gap = candidate.bbox.y - marker_line.bbox.bottom
max_gap = max(marker_line.bbox.h, candidate.bbox.h, 24) * 1.5
return 0 <= vertical_gap <= max_gap
def _find_overlapping_table(tables: list[Table], candidate: Table, *, iou_threshold: float = 0.9) -> Table | None:
for table in tables:
if table.page_index != candidate.page_index:
continue
if bbox_iou(table.bbox, candidate.bbox) >= iou_threshold:
return table
return None
def _average_confidence(lines: list[TextLine]) -> float | None:
confidences = [item.confidence for item in lines if item.confidence is not None]
return sum(confidences) / len(confidences) if confidences else None