"""End-to-end parsing pipeline and file exports.""" from __future__ import annotations import json import shutil from pathlib import Path from .engines import create_engine from .errors import EngineUnavailableError, ParseError from .markdown import render_document_markdown from .quality import evaluate_document_quality from .schemas import Document from .structure import build_document from .tables import write_table_files _CROPPABLE_IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"} def parse_document( input_path: str | Path, output_dir: str | Path, *, engine_name: str = "auto", model_config: str | Path | None = None, fallback_engine: str | None = None, fallback_model_config: str | Path | None = None, low_confidence_threshold: float = 0.80, ) -> Document: source = Path(input_path) out = Path(output_dir) if not source.exists(): raise ParseError(f"Input does not exist: {source}") out.mkdir(parents=True, exist_ok=True) engine_kwargs = {"model_config": model_config} if engine_name in {"candidate", "ours"} else {} engine = create_engine(engine_name, **engine_kwargs) engine_output = engine.recognize(source) avg_confidence = engine_output.average_confidence fallback_decision = _fallback_decision( primary_engine=engine_name, fallback_engine=fallback_engine, fallback_model_config=fallback_model_config, primary_average_confidence=avg_confidence, low_confidence_threshold=low_confidence_threshold, ) if fallback_decision is not None: engine_output.metadata["fallback"] = fallback_decision if fallback_decision is not None and fallback_decision["triggered"]: try: fallback_kwargs = {"model_config": fallback_model_config} if fallback_model_config is not None else {} fallback = create_engine(str(fallback_decision["fallback_engine"]), **fallback_kwargs) fallback_output = fallback.recognize(source) fallback_decision["outcome"] = "success" fallback_output.metadata["fallback"] = fallback_decision engine_output = fallback_output except EngineUnavailableError as exc: fallback_decision["outcome"] = "error" fallback_decision["error"] = str(exc) engine_output.metadata["fallback"] = fallback_decision engine_output.metadata["fallback_error"] = str(exc) except Exception as exc: fallback_decision["outcome"] = "error" fallback_decision["error"] = f"{type(exc).__name__}: {exc}" engine_output.metadata["fallback"] = fallback_decision engine_output.metadata["fallback_error"] = f"{type(exc).__name__}: {exc}" document = build_document(str(source), engine_output) quality = evaluate_document_quality(document) document.metadata["quality"] = quality.to_dict() export_document(document, out, source) return document def _fallback_decision( *, primary_engine: str, fallback_engine: str | None, fallback_model_config: str | Path | None, primary_average_confidence: float | None, low_confidence_threshold: float, ) -> dict[str, object] | None: if not fallback_engine and fallback_model_config is None: return None resolved_fallback_engine = fallback_engine or "candidate" decision: dict[str, object] = { "configured": True, "triggered": False, "outcome": "not_triggered", "primary_engine": primary_engine, "fallback_engine": resolved_fallback_engine, "fallback_model_config": str(fallback_model_config) if fallback_model_config is not None else None, "primary_average_confidence": primary_average_confidence, "threshold": low_confidence_threshold, } if fallback_model_config is None and resolved_fallback_engine.lower() == primary_engine.lower(): decision["reason"] = "same_engine" return decision if primary_average_confidence is None: decision["reason"] = "confidence_not_available" return decision if primary_average_confidence >= low_confidence_threshold: decision["reason"] = "confidence_above_threshold" return decision decision["triggered"] = True decision["outcome"] = "triggered" decision["reason"] = "average_confidence_below_threshold" return decision def export_document(document: Document, output_dir: Path, source_path: Path) -> None: tables_dir = output_dir / "tables" figures_dir = output_dir / "figures" for table in document.tables: write_table_files(table, tables_dir) figures_dir.mkdir(parents=True, exist_ok=True) for figure in document.figures: _export_figure_image(figure, figures_dir, source_path) (figures_dir / "metadata.json").write_text( json.dumps([figure.to_dict() for figure in document.figures], ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) quality = document.metadata.get("quality") if isinstance(quality, dict): (output_dir / "quality.json").write_text(json.dumps(quality, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (output_dir / "document.json").write_text( json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) (output_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8") def _export_figure_image(figure: object, figures_dir: Path, source_path: Path) -> None: image_path = getattr(figure, "image_path", None) if image_path: source_figure = Path(str(image_path)) if not source_figure.is_absolute(): source_base = source_path if source_path.is_dir() else source_path.parent source_figure = source_base / source_figure if source_figure.exists(): target = figures_dir / source_figure.name shutil.copy2(source_figure, target) setattr(figure, "image_path", str(target)) return _record_figure_export_warning(figure, f"figure image_path does not exist: {source_figure}") _crop_figure_from_source(figure, figures_dir, source_path) def _crop_figure_from_source(figure: object, figures_dir: Path, source_path: Path) -> None: source_image = _source_image_for_figure(figure, source_path) if source_image is None: return try: from PIL import Image except ImportError as exc: _record_figure_export_warning(figure, f"Pillow is required to crop figure image: {exc}") return try: bbox = getattr(figure, "bbox") left = max(0, int(round(bbox.x))) top = max(0, int(round(bbox.y))) right = max(left, int(round(bbox.right))) bottom = max(top, int(round(bbox.bottom))) with Image.open(source_image) as image: right = min(right, image.width) bottom = min(bottom, image.height) if right <= left or bottom <= top: _record_figure_export_warning(figure, f"figure bbox is outside source image bounds: {bbox.to_list()}") return target = figures_dir / f"{getattr(figure, 'figure_id', 'figure')}.png" image.crop((left, top, right, bottom)).save(target) setattr(figure, "image_path", str(target)) metadata = getattr(figure, "metadata") if isinstance(metadata, dict): metadata["export_source"] = "source_crop" except Exception as exc: _record_figure_export_warning(figure, f"{type(exc).__name__}: {exc}") def _source_image_for_figure(figure: object, source_path: Path) -> Path | None: if source_path.is_dir(): page_paths = [ path for path in sorted(source_path.iterdir()) if path.is_file() and path.suffix.lower() in _CROPPABLE_IMAGE_SUFFIXES ] if not page_paths: _record_figure_export_warning(figure, f"page bundle has no croppable image files: {source_path}") return None page_index = getattr(figure, "page_index", 0) if not isinstance(page_index, int) or page_index < 0 or page_index >= len(page_paths): _record_figure_export_warning(figure, f"figure page_index is outside page bundle bounds: {page_index}") return None return page_paths[page_index] if source_path.suffix.lower() not in _CROPPABLE_IMAGE_SUFFIXES: _record_figure_export_warning(figure, f"source image format cannot be cropped: {source_path.suffix or ''}") return None return source_path def _record_figure_export_warning(figure: object, message: str) -> None: metadata = getattr(figure, "metadata", None) if isinstance(metadata, dict): warnings = metadata.setdefault("export_warnings", []) if isinstance(warnings, list): warnings.append(message)