Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/scripts/run_public_table100_claim.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 22.7 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/scripts/run_public_table100_claim.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/scripts/run_public_table100_claim.py
-
curl -L -o run_public_table100_claim.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/scripts/run_public_table100_claim.py
22.7 kB
| #!/usr/bin/env python3 | |
| """Run the 100-page public OmniDocBench table claim benchmark.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import shutil | |
| import subprocess | |
| import sys | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from ocrtech.benchmark import BenchmarkResult, _quality_metrics, _write_reports, run_benchmark, summarize_benchmark_report | |
| from ocrtech.manifest import ManifestEntry, load_manifest | |
| from ocrtech.references import load_reference, score_document | |
| from ocrtech.schemas import Document | |
| from ocrtech.table_analysis import analyze_table_cells | |
| from ocrtech.validation import validate_claim | |
| def build_parser() -> argparse.ArgumentParser: | |
| parser = argparse.ArgumentParser(description="Run public OmniDocBench table100 candidate-vs-Surya evidence.") | |
| parser.add_argument( | |
| "--manifest", | |
| default="data/public/omnidocbench-table100/manifests/omnidocbench-eval.jsonl", | |
| help="audited OmniDocBench eval manifest", | |
| ) | |
| parser.add_argument( | |
| "--candidate-model-config", | |
| default="outputs/figure-caption-table-lexicon-normalize-width-guard-v4/model-card.json", | |
| help="candidate model card", | |
| ) | |
| parser.add_argument("--out", default="outputs/experiment-public-omnidocbench-table100-width-guard-v4") | |
| parser.add_argument("--validation-config", default="configs/validation-public-omnidocbench-surya.json") | |
| parser.add_argument("--baselines", default="candidate,surya") | |
| parser.add_argument("--skip-benchmark", action="store_true", help="reuse an existing benchmark/report.json") | |
| parser.add_argument("--no-resume-existing", action="store_true", help="ignore existing document.json outputs") | |
| parser.add_argument("--sample-timeout-seconds", type=int, default=300) | |
| parser.add_argument("--in-process", action="store_true", help="run the benchmark in one Python process instead of isolated page subprocesses") | |
| parser.add_argument( | |
| "--status-only", | |
| action="store_true", | |
| help="write run-status.json and run-status.md from the current checkpoint without running benchmark, summary, validation, or analysis", | |
| ) | |
| return parser | |
| def main() -> int: | |
| args = build_parser().parse_args() | |
| manifest_path = Path(args.manifest) | |
| output_root = Path(args.out) | |
| benchmark_dir = output_root / "benchmark" | |
| report_path = benchmark_dir / "report.json" | |
| baselines = [item.strip() for item in args.baselines.split(",") if item.strip()] | |
| if not baselines: | |
| raise ValueError("--baselines must include at least one baseline") | |
| entries = load_manifest(manifest_path) | |
| input_paths = [entry.image_path for entry in entries] | |
| print(f"manifest={manifest_path} samples={len(entries)}", flush=True) | |
| print(f"benchmark_out={benchmark_dir}", flush=True) | |
| print(f"baselines={','.join(baselines)}", flush=True) | |
| print(f"isolated={not args.in_process} resume_existing={not args.no_resume_existing}", flush=True) | |
| if args.status_only: | |
| status = _write_interim_status(output_root, report_path, entries, baselines) | |
| print(f"status={output_root / 'run-status.json'} rows={status['rows']} expected_rows={status['expected_rows']}", flush=True) | |
| return 0 | |
| if not args.skip_benchmark: | |
| if args.in_process: | |
| run_benchmark( | |
| input_paths, | |
| benchmark_dir, | |
| baselines=baselines, | |
| eval_manifest=manifest_path, | |
| candidate_model_config=args.candidate_model_config, | |
| resume_existing=not args.no_resume_existing, | |
| sample_timeout_seconds=args.sample_timeout_seconds, | |
| ) | |
| else: | |
| _run_isolated_benchmark( | |
| entries, | |
| benchmark_dir, | |
| baselines=baselines, | |
| manifest_path=manifest_path, | |
| candidate_model_config=Path(args.candidate_model_config), | |
| resume_existing=not args.no_resume_existing, | |
| sample_timeout_seconds=args.sample_timeout_seconds, | |
| ) | |
| elif not report_path.exists(): | |
| raise FileNotFoundError(f"--skip-benchmark requested but report is missing: {report_path}") | |
| summary = summarize_benchmark_report( | |
| report_path, | |
| output_root / "summary", | |
| candidate="candidate", | |
| baselines=[item for item in baselines if item != "candidate"], | |
| ) | |
| validation = validate_claim( | |
| report_path, | |
| output_root / "validation", | |
| config_path=args.validation_config, | |
| eval_manifest=manifest_path, | |
| ) | |
| table_analysis = analyze_table_cells( | |
| report_path, | |
| manifest_path, | |
| output_root / "table-cell-analysis", | |
| candidate="candidate", | |
| baselines=[item for item in baselines if item != "candidate"], | |
| ) | |
| print(f"report={report_path}", flush=True) | |
| print(f"summary={output_root / 'summary' / 'summary.json'} paired={len(summary.paired_metrics)}", flush=True) | |
| print(f"validation={validation.claim_status} passed={validation.passed}", flush=True) | |
| print( | |
| f"table_cell_analysis={output_root / 'table-cell-analysis' / 'table-cell-analysis.json'} " | |
| f"systems={len(table_analysis.systems)}", | |
| flush=True, | |
| ) | |
| return 0 if validation.passed else 1 | |
| def _run_isolated_benchmark( | |
| entries: list[ManifestEntry], | |
| benchmark_dir: Path, | |
| *, | |
| baselines: list[str], | |
| manifest_path: Path, | |
| candidate_model_config: Path, | |
| resume_existing: bool, | |
| sample_timeout_seconds: int, | |
| ) -> list[BenchmarkResult]: | |
| benchmark_dir.mkdir(parents=True, exist_ok=True) | |
| status_path = benchmark_dir.parent / "run-status.json" | |
| tmp_root = benchmark_dir.parent / "_isolated-tmp" | |
| results: list[BenchmarkResult] = _load_existing_report(benchmark_dir / "report.json") | |
| completed = {(result.baseline, result.input_path) for result in results} | |
| entry_by_path = {str(Path(entry.image_path)): entry for entry in entries} | |
| for baseline in baselines: | |
| for entry in entries: | |
| input_path = Path(entry.image_path) | |
| key = (baseline, str(input_path)) | |
| if key in completed: | |
| continue | |
| _write_run_status( | |
| status_path, | |
| baseline=baseline, | |
| sample_id=entry.sample_id, | |
| input_path=input_path, | |
| rows=len(results), | |
| status="running", | |
| ) | |
| existing_document = benchmark_dir / baseline / input_path.stem / "document.json" | |
| if resume_existing and existing_document.exists(): | |
| result = _score_existing_document(baseline, input_path, entry, existing_document) | |
| else: | |
| result = _run_one_isolated( | |
| baseline, | |
| input_path, | |
| manifest_path, | |
| benchmark_dir, | |
| tmp_root, | |
| candidate_model_config=candidate_model_config, | |
| entry=entry_by_path[str(input_path)], | |
| sample_timeout_seconds=sample_timeout_seconds, | |
| ) | |
| results.append(result) | |
| completed.add(key) | |
| _write_reports(results, benchmark_dir) | |
| _write_run_status( | |
| status_path, | |
| baseline=baseline, | |
| sample_id=entry.sample_id, | |
| input_path=input_path, | |
| rows=len(results), | |
| status=result.status, | |
| error=result.error, | |
| ) | |
| print( | |
| f"progress baseline={baseline} sample={entry.sample_id} status={result.status} rows={len(results)}", | |
| flush=True, | |
| ) | |
| return results | |
| def _write_run_status( | |
| path: Path, | |
| *, | |
| baseline: str, | |
| sample_id: str, | |
| input_path: Path, | |
| rows: int, | |
| status: str, | |
| error: str | None = None, | |
| ) -> None: | |
| payload = { | |
| "updated_at_utc": datetime.now(timezone.utc).isoformat(), | |
| "baseline": baseline, | |
| "sample_id": sample_id, | |
| "input_path": str(input_path), | |
| "rows": rows, | |
| "status": status, | |
| "error": error, | |
| } | |
| path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| def _write_interim_status( | |
| output_root: Path, | |
| report_path: Path, | |
| entries: list[ManifestEntry], | |
| baselines: list[str], | |
| ) -> dict[str, object]: | |
| results = _load_existing_report(report_path) | |
| completed = {(result.baseline, result.input_path) for result in results} | |
| status_counts: dict[str, int] = {} | |
| baseline_counts: dict[str, int] = {} | |
| per_baseline_progress: dict[str, dict[str, int]] = { | |
| baseline: {"expected": len(entries), "completed": 0, "ok": 0, "error": 0, "missing": len(entries)} | |
| for baseline in baselines | |
| } | |
| for result in results: | |
| status_counts[result.status] = status_counts.get(result.status, 0) + 1 | |
| baseline_counts[result.baseline] = baseline_counts.get(result.baseline, 0) + 1 | |
| progress = per_baseline_progress.setdefault( | |
| result.baseline, | |
| {"expected": len(entries), "completed": 0, "ok": 0, "error": 0, "missing": len(entries)}, | |
| ) | |
| progress["completed"] += 1 | |
| if result.status == "ok": | |
| progress["ok"] += 1 | |
| elif result.status == "error": | |
| progress["error"] += 1 | |
| progress["missing"] = max(0, progress["expected"] - progress["completed"]) | |
| last_result: dict[str, object] | None = None | |
| if results: | |
| result = results[-1] | |
| last_result = { | |
| "baseline": result.baseline, | |
| "sample_id": result.sample_id, | |
| "input_path": result.input_path, | |
| "status": result.status, | |
| "error": result.error, | |
| } | |
| error_results = [result for result in results if result.status == "error"] | |
| recent_errors = [ | |
| { | |
| "baseline": result.baseline, | |
| "sample_id": result.sample_id, | |
| "input_path": result.input_path, | |
| "error": result.error, | |
| } | |
| for result in error_results[-10:] | |
| ] | |
| next_item: dict[str, object] | None = None | |
| for baseline in baselines: | |
| for entry in entries: | |
| key = (baseline, str(Path(entry.image_path))) | |
| if key in completed: | |
| continue | |
| next_item = { | |
| "baseline": baseline, | |
| "sample_id": entry.sample_id, | |
| "input_path": str(Path(entry.image_path)), | |
| } | |
| break | |
| if next_item is not None: | |
| break | |
| final_artifacts = { | |
| "summary": (output_root / "summary" / "summary.json").exists(), | |
| "validation": (output_root / "validation" / "validation.json").exists(), | |
| "table_cell_analysis": (output_root / "table-cell-analysis" / "table-cell-analysis.json").exists(), | |
| } | |
| validation_status = _load_validation_status(output_root / "validation" / "validation.json") | |
| expected_rows = len(entries) * len(baselines) | |
| current_phase = str(next_item["baseline"]) if next_item is not None else "finalizing" if not all(final_artifacts.values()) else "complete" | |
| evidence_package_ready = len(completed) >= expected_rows and all(final_artifacts.values()) | |
| validation_passed = validation_status.get("passed") is True | |
| payload: dict[str, object] = { | |
| "updated_at_utc": datetime.now(timezone.utc).isoformat(), | |
| "status": "complete" if evidence_package_ready else "incomplete", | |
| "rows": len(results), | |
| "expected_rows": expected_rows, | |
| "baselines": baseline_counts, | |
| "baseline_progress": per_baseline_progress, | |
| "current_phase": current_phase, | |
| "row_statuses": status_counts, | |
| "last_result": last_result, | |
| "error_count": len(error_results), | |
| "recent_errors": recent_errors, | |
| "next": next_item, | |
| "final_artifacts": final_artifacts, | |
| "evidence_package_ready": evidence_package_ready, | |
| "validation": validation_status, | |
| "paper_claim_ready": evidence_package_ready and validation_passed, | |
| } | |
| (output_root / "run-status.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_interim_status_markdown(output_root / "run-status.md", payload) | |
| return payload | |
| def _load_validation_status(validation_path: Path) -> dict[str, object]: | |
| if not validation_path.exists(): | |
| return { | |
| "present": False, | |
| "claim_status": None, | |
| "passed": None, | |
| } | |
| payload = json.loads(validation_path.read_text(encoding="utf-8")) | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"validation report must be an object: {validation_path}") | |
| return { | |
| "present": True, | |
| "claim_status": payload.get("claim_status"), | |
| "passed": payload.get("passed"), | |
| "missing_requirements": payload.get("missing_requirements", []), | |
| } | |
| def _write_interim_status_markdown(path: Path, payload: dict[str, object]) -> None: | |
| baseline_progress = payload.get("baseline_progress") | |
| final_artifacts = payload.get("final_artifacts") | |
| recent_errors = payload.get("recent_errors") | |
| validation_status = payload.get("validation") | |
| lines = [ | |
| "# Public Table100 Run Status", | |
| "", | |
| f"Updated: `{payload.get('updated_at_utc')}`", | |
| f"Status: `{payload.get('status')}`", | |
| f"Current phase: `{payload.get('current_phase')}`", | |
| f"Rows: `{payload.get('rows')}` / `{payload.get('expected_rows')}`", | |
| f"Evidence package ready: `{'yes' if payload.get('evidence_package_ready') else 'no'}`", | |
| f"Paper claim ready: `{'yes' if payload.get('paper_claim_ready') else 'no'}`", | |
| "", | |
| "## Baseline Progress", | |
| "", | |
| "| baseline | expected | completed | ok | error | missing |", | |
| "| --- | ---: | ---: | ---: | ---: | ---: |", | |
| ] | |
| if isinstance(baseline_progress, dict): | |
| for baseline, progress in sorted(baseline_progress.items()): | |
| if not isinstance(progress, dict): | |
| continue | |
| lines.append( | |
| "| {baseline} | {expected} | {completed} | {ok} | {error} | {missing} |".format( | |
| baseline=baseline, | |
| expected=progress.get("expected", ""), | |
| completed=progress.get("completed", ""), | |
| ok=progress.get("ok", ""), | |
| error=progress.get("error", ""), | |
| missing=progress.get("missing", ""), | |
| ) | |
| ) | |
| lines.extend(["", "## Last Result", ""]) | |
| last_result = payload.get("last_result") | |
| if isinstance(last_result, dict): | |
| lines.extend( | |
| [ | |
| f"- baseline: `{last_result.get('baseline')}`", | |
| f"- sample_id: `{last_result.get('sample_id')}`", | |
| f"- status: `{last_result.get('status')}`", | |
| f"- error: `{last_result.get('error')}`", | |
| ] | |
| ) | |
| else: | |
| lines.append("- none") | |
| lines.extend(["", "## Recent Errors", ""]) | |
| if isinstance(recent_errors, list) and recent_errors: | |
| for error in recent_errors: | |
| if not isinstance(error, dict): | |
| continue | |
| lines.append( | |
| "- `{baseline}` / `{sample_id}`: {message}".format( | |
| baseline=error.get("baseline"), | |
| sample_id=error.get("sample_id"), | |
| message=error.get("error"), | |
| ) | |
| ) | |
| else: | |
| lines.append("- none") | |
| lines.extend(["", "## Final Artifacts", ""]) | |
| if isinstance(final_artifacts, dict): | |
| for name, exists in sorted(final_artifacts.items()): | |
| lines.append(f"- {name}: `{'present' if exists else 'missing'}`") | |
| else: | |
| lines.append("- unknown") | |
| lines.extend(["", "## Validation", ""]) | |
| if isinstance(validation_status, dict): | |
| lines.extend( | |
| [ | |
| f"- present: `{'yes' if validation_status.get('present') else 'no'}`", | |
| f"- claim_status: `{validation_status.get('claim_status')}`", | |
| f"- passed: `{validation_status.get('passed')}`", | |
| ] | |
| ) | |
| missing_requirements = validation_status.get("missing_requirements") | |
| if isinstance(missing_requirements, list) and missing_requirements: | |
| lines.append("- missing requirements:") | |
| for requirement in missing_requirements: | |
| lines.append(f" - {requirement}") | |
| else: | |
| lines.append("- unknown") | |
| next_item = payload.get("next") | |
| lines.extend(["", "## Next", ""]) | |
| if isinstance(next_item, dict): | |
| lines.extend( | |
| [ | |
| f"- baseline: `{next_item.get('baseline')}`", | |
| f"- sample_id: `{next_item.get('sample_id')}`", | |
| f"- input_path: `{next_item.get('input_path')}`", | |
| ] | |
| ) | |
| else: | |
| lines.append("- none") | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def _load_existing_report(report_path: Path) -> list[BenchmarkResult]: | |
| if not report_path.exists(): | |
| return [] | |
| payload = json.loads(report_path.read_text(encoding="utf-8")) | |
| if not isinstance(payload, list): | |
| raise ValueError(f"benchmark report must be a list: {report_path}") | |
| by_key: dict[tuple[str, str], BenchmarkResult] = {} | |
| for row in payload: | |
| if not isinstance(row, dict): | |
| raise ValueError(f"benchmark report row must be an object: {report_path}") | |
| result = BenchmarkResult( | |
| baseline=str(row.get("baseline") or ""), | |
| input_path=str(row.get("input_path") or ""), | |
| status=str(row.get("status") or "error"), | |
| latency_seconds=float(row.get("latency_seconds") or 0.0), | |
| sample_id=str(row.get("sample_id") or "") or None, | |
| slices=[str(item) for item in row.get("slices", [])] if isinstance(row.get("slices"), list) else [], | |
| metrics={str(key): float(value) for key, value in dict(row.get("metrics") or {}).items()}, | |
| error=str(row.get("error") or "") or None, | |
| ) | |
| by_key[(result.baseline, result.input_path)] = result | |
| return list(by_key.values()) | |
| def _score_existing_document(baseline: str, input_path: Path, entry: ManifestEntry, document_path: Path) -> BenchmarkResult: | |
| payload = json.loads(document_path.read_text(encoding="utf-8")) | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"document JSON must be an object: {document_path}") | |
| document = Document.from_dict(payload) | |
| reference_path = entry.metadata.get("reference_path") | |
| explicit_reference_path = str(reference_path) if isinstance(reference_path, str) else None | |
| metrics = score_document(document, document.text, load_reference(input_path, None, explicit_path=explicit_reference_path)) | |
| metrics.update(_quality_metrics(document)) | |
| return BenchmarkResult( | |
| baseline=baseline, | |
| input_path=str(input_path), | |
| status="ok", | |
| latency_seconds=0.0, | |
| sample_id=entry.sample_id, | |
| slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [], | |
| metrics=metrics, | |
| ) | |
| def _run_one_isolated( | |
| baseline: str, | |
| input_path: Path, | |
| manifest_path: Path, | |
| benchmark_dir: Path, | |
| tmp_root: Path, | |
| *, | |
| candidate_model_config: Path, | |
| entry: ManifestEntry, | |
| sample_timeout_seconds: int, | |
| ) -> BenchmarkResult: | |
| tmp_dir = tmp_root / baseline / input_path.stem | |
| if tmp_dir.exists(): | |
| shutil.rmtree(tmp_dir) | |
| tmp_dir.mkdir(parents=True) | |
| command = [ | |
| sys.executable, | |
| "-m", | |
| "ocrtech.cli", | |
| "benchmark", | |
| str(input_path), | |
| "--out", | |
| str(tmp_dir), | |
| "--baselines", | |
| baseline, | |
| "--eval-manifest", | |
| str(manifest_path), | |
| ] | |
| if baseline == "candidate": | |
| command.extend(["--candidate-model-config", str(candidate_model_config)]) | |
| try: | |
| completed = subprocess.run( | |
| command, | |
| check=False, | |
| capture_output=True, | |
| text=True, | |
| timeout=sample_timeout_seconds, | |
| ) | |
| except subprocess.TimeoutExpired as exc: | |
| return BenchmarkResult( | |
| baseline=baseline, | |
| input_path=str(input_path), | |
| status="error", | |
| latency_seconds=float(sample_timeout_seconds), | |
| sample_id=entry.sample_id, | |
| slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [], | |
| error=f"TimeoutExpired: timed out after {sample_timeout_seconds}s", | |
| ) | |
| tmp_report = tmp_dir / "report.json" | |
| if not tmp_report.exists(): | |
| stderr_tail = (completed.stderr or completed.stdout or "").strip().splitlines()[-5:] | |
| return BenchmarkResult( | |
| baseline=baseline, | |
| input_path=str(input_path), | |
| status="error", | |
| latency_seconds=0.0, | |
| sample_id=entry.sample_id, | |
| slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [], | |
| error=f"isolated benchmark exited {completed.returncode}: {' | '.join(stderr_tail)}", | |
| ) | |
| rows = json.loads(tmp_report.read_text(encoding="utf-8")) | |
| if not isinstance(rows, list) or len(rows) != 1 or not isinstance(rows[0], dict): | |
| raise ValueError(f"isolated benchmark report must contain exactly one row: {tmp_report}") | |
| output_source = tmp_dir / baseline / input_path.stem | |
| output_target = benchmark_dir / baseline / input_path.stem | |
| if output_source.exists(): | |
| if output_target.exists(): | |
| shutil.rmtree(output_target) | |
| shutil.copytree(output_source, output_target) | |
| row = rows[0] | |
| return BenchmarkResult( | |
| baseline=str(row.get("baseline") or baseline), | |
| input_path=str(row.get("input_path") or input_path), | |
| status=str(row.get("status") or "error"), | |
| latency_seconds=float(row.get("latency_seconds") or 0.0), | |
| sample_id=str(row.get("sample_id") or entry.sample_id) or None, | |
| slices=[str(item) for item in row.get("slices", [])] if isinstance(row.get("slices"), list) else [], | |
| metrics={str(key): float(value) for key, value in dict(row.get("metrics") or {}).items()}, | |
| error=str(row.get("error") or "") or None, | |
| ) | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |