Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download src/ocrtech/pipeline.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 8.96 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/pipeline.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/src/ocrtech/pipeline.py
-
curl -L -o pipeline.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/pipeline.py
8.96 kB
| """End-to-end parsing pipeline and file exports.""" | |
| from __future__ import annotations | |
| import json | |
| import shutil | |
| from pathlib import Path | |
| from .engines import create_engine | |
| from .errors import EngineUnavailableError, ParseError | |
| from .markdown import render_document_markdown | |
| from .quality import evaluate_document_quality | |
| from .schemas import Document | |
| from .structure import build_document | |
| from .tables import write_table_files | |
| _CROPPABLE_IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"} | |
| def parse_document( | |
| input_path: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| engine_name: str = "auto", | |
| model_config: str | Path | None = None, | |
| fallback_engine: str | None = None, | |
| fallback_model_config: str | Path | None = None, | |
| low_confidence_threshold: float = 0.80, | |
| ) -> Document: | |
| source = Path(input_path) | |
| out = Path(output_dir) | |
| if not source.exists(): | |
| raise ParseError(f"Input does not exist: {source}") | |
| out.mkdir(parents=True, exist_ok=True) | |
| engine_kwargs = {"model_config": model_config} if engine_name in {"candidate", "ours"} else {} | |
| engine = create_engine(engine_name, **engine_kwargs) | |
| engine_output = engine.recognize(source) | |
| avg_confidence = engine_output.average_confidence | |
| fallback_decision = _fallback_decision( | |
| primary_engine=engine_name, | |
| fallback_engine=fallback_engine, | |
| fallback_model_config=fallback_model_config, | |
| primary_average_confidence=avg_confidence, | |
| low_confidence_threshold=low_confidence_threshold, | |
| ) | |
| if fallback_decision is not None: | |
| engine_output.metadata["fallback"] = fallback_decision | |
| if fallback_decision is not None and fallback_decision["triggered"]: | |
| try: | |
| fallback_kwargs = {"model_config": fallback_model_config} if fallback_model_config is not None else {} | |
| fallback = create_engine(str(fallback_decision["fallback_engine"]), **fallback_kwargs) | |
| fallback_output = fallback.recognize(source) | |
| fallback_decision["outcome"] = "success" | |
| fallback_output.metadata["fallback"] = fallback_decision | |
| engine_output = fallback_output | |
| except EngineUnavailableError as exc: | |
| fallback_decision["outcome"] = "error" | |
| fallback_decision["error"] = str(exc) | |
| engine_output.metadata["fallback"] = fallback_decision | |
| engine_output.metadata["fallback_error"] = str(exc) | |
| except Exception as exc: | |
| fallback_decision["outcome"] = "error" | |
| fallback_decision["error"] = f"{type(exc).__name__}: {exc}" | |
| engine_output.metadata["fallback"] = fallback_decision | |
| engine_output.metadata["fallback_error"] = f"{type(exc).__name__}: {exc}" | |
| document = build_document(str(source), engine_output) | |
| quality = evaluate_document_quality(document) | |
| document.metadata["quality"] = quality.to_dict() | |
| export_document(document, out, source) | |
| return document | |
| def _fallback_decision( | |
| *, | |
| primary_engine: str, | |
| fallback_engine: str | None, | |
| fallback_model_config: str | Path | None, | |
| primary_average_confidence: float | None, | |
| low_confidence_threshold: float, | |
| ) -> dict[str, object] | None: | |
| if not fallback_engine and fallback_model_config is None: | |
| return None | |
| resolved_fallback_engine = fallback_engine or "candidate" | |
| decision: dict[str, object] = { | |
| "configured": True, | |
| "triggered": False, | |
| "outcome": "not_triggered", | |
| "primary_engine": primary_engine, | |
| "fallback_engine": resolved_fallback_engine, | |
| "fallback_model_config": str(fallback_model_config) if fallback_model_config is not None else None, | |
| "primary_average_confidence": primary_average_confidence, | |
| "threshold": low_confidence_threshold, | |
| } | |
| if fallback_model_config is None and resolved_fallback_engine.lower() == primary_engine.lower(): | |
| decision["reason"] = "same_engine" | |
| return decision | |
| if primary_average_confidence is None: | |
| decision["reason"] = "confidence_not_available" | |
| return decision | |
| if primary_average_confidence >= low_confidence_threshold: | |
| decision["reason"] = "confidence_above_threshold" | |
| return decision | |
| decision["triggered"] = True | |
| decision["outcome"] = "triggered" | |
| decision["reason"] = "average_confidence_below_threshold" | |
| return decision | |
| def export_document(document: Document, output_dir: Path, source_path: Path) -> None: | |
| tables_dir = output_dir / "tables" | |
| figures_dir = output_dir / "figures" | |
| for table in document.tables: | |
| write_table_files(table, tables_dir) | |
| figures_dir.mkdir(parents=True, exist_ok=True) | |
| for figure in document.figures: | |
| _export_figure_image(figure, figures_dir, source_path) | |
| (figures_dir / "metadata.json").write_text( | |
| json.dumps([figure.to_dict() for figure in document.figures], ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| quality = document.metadata.get("quality") | |
| if isinstance(quality, dict): | |
| (output_dir / "quality.json").write_text(json.dumps(quality, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (output_dir / "document.json").write_text( | |
| json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| (output_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") | |
| def _export_figure_image(figure: object, figures_dir: Path, source_path: Path) -> None: | |
| image_path = getattr(figure, "image_path", None) | |
| if image_path: | |
| source_figure = Path(str(image_path)) | |
| if not source_figure.is_absolute(): | |
| source_base = source_path if source_path.is_dir() else source_path.parent | |
| source_figure = source_base / source_figure | |
| if source_figure.exists(): | |
| target = figures_dir / source_figure.name | |
| shutil.copy2(source_figure, target) | |
| setattr(figure, "image_path", str(target)) | |
| return | |
| _record_figure_export_warning(figure, f"figure image_path does not exist: {source_figure}") | |
| _crop_figure_from_source(figure, figures_dir, source_path) | |
| def _crop_figure_from_source(figure: object, figures_dir: Path, source_path: Path) -> None: | |
| source_image = _source_image_for_figure(figure, source_path) | |
| if source_image is None: | |
| return | |
| try: | |
| from PIL import Image | |
| except ImportError as exc: | |
| _record_figure_export_warning(figure, f"Pillow is required to crop figure image: {exc}") | |
| return | |
| try: | |
| bbox = getattr(figure, "bbox") | |
| left = max(0, int(round(bbox.x))) | |
| top = max(0, int(round(bbox.y))) | |
| right = max(left, int(round(bbox.right))) | |
| bottom = max(top, int(round(bbox.bottom))) | |
| with Image.open(source_image) as image: | |
| right = min(right, image.width) | |
| bottom = min(bottom, image.height) | |
| if right <= left or bottom <= top: | |
| _record_figure_export_warning(figure, f"figure bbox is outside source image bounds: {bbox.to_list()}") | |
| return | |
| target = figures_dir / f"{getattr(figure, 'figure_id', 'figure')}.png" | |
| image.crop((left, top, right, bottom)).save(target) | |
| setattr(figure, "image_path", str(target)) | |
| metadata = getattr(figure, "metadata") | |
| if isinstance(metadata, dict): | |
| metadata["export_source"] = "source_crop" | |
| except Exception as exc: | |
| _record_figure_export_warning(figure, f"{type(exc).__name__}: {exc}") | |
| def _source_image_for_figure(figure: object, source_path: Path) -> Path | None: | |
| if source_path.is_dir(): | |
| page_paths = [ | |
| path | |
| for path in sorted(source_path.iterdir()) | |
| if path.is_file() and path.suffix.lower() in _CROPPABLE_IMAGE_SUFFIXES | |
| ] | |
| if not page_paths: | |
| _record_figure_export_warning(figure, f"page bundle has no croppable image files: {source_path}") | |
| return None | |
| page_index = getattr(figure, "page_index", 0) | |
| if not isinstance(page_index, int) or page_index < 0 or page_index >= len(page_paths): | |
| _record_figure_export_warning(figure, f"figure page_index is outside page bundle bounds: {page_index}") | |
| return None | |
| return page_paths[page_index] | |
| if source_path.suffix.lower() not in _CROPPABLE_IMAGE_SUFFIXES: | |
| _record_figure_export_warning(figure, f"source image format cannot be cropped: {source_path.suffix or '<none>'}") | |
| return None | |
| return source_path | |
| def _record_figure_export_warning(figure: object, message: str) -> None: | |
| metadata = getattr(figure, "metadata", None) | |
| if isinstance(metadata, dict): | |
| warnings = metadata.setdefault("export_warnings", []) | |
| if isinstance(warnings, list): | |
| warnings.append(message) | |