Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/external_outputs.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 7.38 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/external_outputs.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/external_outputs.py
-
curl -L -o external_outputs.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/external_outputs.py
7.38 kB
| """Helpers for normalizing external OCR baseline outputs.""" | |
| from __future__ import annotations | |
| import json | |
| import re | |
| from pathlib import Path | |
| from typing import Any | |
| from .schemas import BBox, Block, Document, Page, Table, TableCell, TextLine | |
| _PAGE_BREAK_PATTERN = re.compile(r"(?:\n\s*<!--\s*pagebreak\s*-->\s*\n)|\f", re.IGNORECASE) | |
| _TABLE_SEPARATOR_PATTERN = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*(?:\|\s*:?-{3,}:?\s*)+\|?\s*$") | |
| def bundle_page_paths(input_path: Path) -> list[Path]: | |
| """Return page image paths for a directory bundle or the single input path.""" | |
| if not input_path.is_dir(): | |
| return [input_path] | |
| allowed_suffixes = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"} | |
| page_paths = [path for path in sorted(input_path.iterdir()) if path.is_file() and path.suffix.lower() in allowed_suffixes] | |
| if not page_paths: | |
| raise ValueError(f"page bundle directory has no supported image files: {input_path}") | |
| return page_paths | |
| def materialize_external_markdown( | |
| source_path: Path, | |
| output_dir: Path, | |
| markdown_text: str, | |
| *, | |
| metadata: dict[str, Any] | None = None, | |
| raw_payload: Any | None = None, | |
| ) -> Document: | |
| """Write normalized markdown and document JSON for an external OCR result.""" | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| cleaned = _clean_markdown(markdown_text) | |
| document = document_from_markdown(source_path, cleaned, metadata=metadata) | |
| (output_dir / "document.md").write_text(cleaned + ("\n" if cleaned and not cleaned.endswith("\n") else ""), encoding="utf-8") | |
| (output_dir / "document.json").write_text( | |
| json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| if raw_payload is not None: | |
| (output_dir / "raw-output.json").write_text( | |
| json.dumps(raw_payload, ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| return document | |
| def document_from_markdown(source_path: Path, markdown_text: str, *, metadata: dict[str, Any] | None = None) -> Document: | |
| """Build a minimal structured document from markdown output.""" | |
| pages: list[Page] = [] | |
| tables: list[Table] = [] | |
| markdown_pages = [part.strip("\n") for part in _PAGE_BREAK_PATTERN.split(markdown_text)] or [markdown_text] | |
| for page_index, page_markdown in enumerate(markdown_pages): | |
| page = _page_from_markdown(page_markdown, page_index=page_index, global_tables=tables) | |
| pages.append(page) | |
| return Document(source_path=str(source_path), pages=pages, tables=tables, figures=[], metadata=dict(metadata or {})) | |
| def _clean_markdown(value: str) -> str: | |
| text = value.replace("\r\n", "\n").replace("\r", "\n").strip() | |
| if text.startswith("```") and text.endswith("```"): | |
| lines = text.splitlines() | |
| if len(lines) >= 2: | |
| text = "\n".join(lines[1:-1]).strip() | |
| return text | |
| def _page_from_markdown(markdown_text: str, *, page_index: int, global_tables: list[Table]) -> Page: | |
| text_lines: list[TextLine] = [] | |
| blocks: list[Block] = [] | |
| lines = markdown_text.splitlines() | |
| order = 0 | |
| y = 0.0 | |
| index = 0 | |
| while index < len(lines): | |
| line = lines[index] | |
| stripped = line.strip() | |
| if not stripped: | |
| index += 1 | |
| y += 20 | |
| continue | |
| table_result = _parse_markdown_table(lines, start=index) | |
| if table_result is not None: | |
| table, consumed, table_text = table_result | |
| table.page_index = page_index | |
| global_tables.append(table) | |
| blocks.append( | |
| Block( | |
| block_id=f"p{page_index}-b{order}", | |
| block_type="table", | |
| page_index=page_index, | |
| bbox=table.bbox, | |
| order=order, | |
| text=table_text, | |
| table_id=table.table_id, | |
| ) | |
| ) | |
| order += 1 | |
| y = table.bbox.bottom + 24 | |
| index += consumed | |
| continue | |
| text = _strip_markdown_prefix(stripped) | |
| bbox = BBox(0, y, max(50, len(text) * 9), 22) | |
| line_id = f"p{page_index}-l{len(text_lines)}" | |
| text_lines.append(TextLine(text=text, bbox=bbox, confidence=None, page_index=page_index, line_id=line_id)) | |
| block_type = "title" if stripped.startswith("#") else "caption" if _looks_like_caption(text) else "text" | |
| blocks.append( | |
| Block( | |
| block_id=f"p{page_index}-b{order}", | |
| block_type=block_type, | |
| page_index=page_index, | |
| bbox=bbox, | |
| order=order, | |
| text=text, | |
| line_ids=[line_id], | |
| ) | |
| ) | |
| order += 1 | |
| y += 28 | |
| index += 1 | |
| return Page(page_index=page_index, text_lines=text_lines, blocks=blocks, metadata={"source": "external-markdown"}) | |
| def _parse_markdown_table(lines: list[str], *, start: int) -> tuple[Table, int, str] | None: | |
| if start + 1 >= len(lines): | |
| return None | |
| header_line = lines[start].strip() | |
| separator_line = lines[start + 1].strip() | |
| if "|" not in header_line or not _TABLE_SEPARATOR_PATTERN.match(separator_line): | |
| return None | |
| data_lines: list[str] = [lines[start], lines[start + 1]] | |
| index = start + 2 | |
| while index < len(lines): | |
| candidate = lines[index].strip() | |
| if not candidate or "|" not in candidate: | |
| break | |
| data_lines.append(lines[index]) | |
| index += 1 | |
| rows = [_split_markdown_row(item) for item in data_lines if "|" in item] | |
| if len(rows) < 2: | |
| return None | |
| header = rows[0] | |
| body = rows[2:] if len(rows) > 2 else [] | |
| cells: list[TableCell] = [] | |
| row_count = 0 | |
| if header: | |
| for col_index, value in enumerate(header): | |
| cells.append(TableCell(row=0, col=col_index, text=value)) | |
| row_count = 1 | |
| for body_row in body: | |
| for col_index, value in enumerate(body_row): | |
| cells.append(TableCell(row=row_count, col=col_index, text=value)) | |
| row_count += 1 | |
| col_count = max((len(row) for row in [header, *body]), default=1) | |
| table_height = max(32.0, 28.0 * max(1, row_count)) | |
| table_width = max(120.0, 120.0 * col_count) | |
| table_text = "\n".join(line.rstrip() for line in data_lines) | |
| table = Table( | |
| table_id=f"table-{start}", | |
| page_index=0, | |
| bbox=BBox(0, start * 28.0, table_width, table_height), | |
| cells=cells, | |
| html=None, | |
| caption=None, | |
| metadata={"source": "external-markdown"}, | |
| ) | |
| return table, len(data_lines), table_text | |
| def _split_markdown_row(line: str) -> list[str]: | |
| trimmed = line.strip() | |
| if trimmed.startswith("|"): | |
| trimmed = trimmed[1:] | |
| if trimmed.endswith("|"): | |
| trimmed = trimmed[:-1] | |
| return [part.strip() for part in trimmed.split("|")] | |
| def _strip_markdown_prefix(text: str) -> str: | |
| value = text.lstrip("#").strip() if text.startswith("#") else text | |
| if value.startswith("- "): | |
| return value[2:].strip() | |
| if value.startswith("* "): | |
| return value[2:].strip() | |
| return value | |
| def _looks_like_caption(text: str) -> bool: | |
| lowered = text.casefold() | |
| return lowered.startswith("figure ") or lowered.startswith("fig. ") or lowered.startswith("चित्र ") | |