#!/usr/bin/env python3 """Run the 100-page public OmniDocBench table claim benchmark.""" from __future__ import annotations import argparse import json import shutil import subprocess import sys from datetime import datetime, timezone from pathlib import Path from ocrtech.benchmark import BenchmarkResult, _quality_metrics, _write_reports, run_benchmark, summarize_benchmark_report from ocrtech.manifest import ManifestEntry, load_manifest from ocrtech.references import load_reference, score_document from ocrtech.schemas import Document from ocrtech.table_analysis import analyze_table_cells from ocrtech.validation import validate_claim def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Run public OmniDocBench table100 candidate-vs-Surya evidence.") parser.add_argument( "--manifest", default="data/public/omnidocbench-table100/manifests/omnidocbench-eval.jsonl", help="audited OmniDocBench eval manifest", ) parser.add_argument( "--candidate-model-config", default="outputs/figure-caption-table-lexicon-normalize-width-guard-v4/model-card.json", help="candidate model card", ) parser.add_argument("--out", default="outputs/experiment-public-omnidocbench-table100-width-guard-v4") parser.add_argument("--validation-config", default="configs/validation-public-omnidocbench-surya.json") parser.add_argument("--baselines", default="candidate,surya") parser.add_argument("--skip-benchmark", action="store_true", help="reuse an existing benchmark/report.json") parser.add_argument("--no-resume-existing", action="store_true", help="ignore existing document.json outputs") parser.add_argument("--sample-timeout-seconds", type=int, default=300) parser.add_argument("--in-process", action="store_true", help="run the benchmark in one Python process instead of isolated page subprocesses") parser.add_argument( "--status-only", action="store_true", help="write run-status.json and run-status.md from the current checkpoint without running benchmark, summary, validation, or analysis", ) return parser def main() -> int: args = build_parser().parse_args() manifest_path = Path(args.manifest) output_root = Path(args.out) benchmark_dir = output_root / "benchmark" report_path = benchmark_dir / "report.json" baselines = [item.strip() for item in args.baselines.split(",") if item.strip()] if not baselines: raise ValueError("--baselines must include at least one baseline") entries = load_manifest(manifest_path) input_paths = [entry.image_path for entry in entries] print(f"manifest={manifest_path} samples={len(entries)}", flush=True) print(f"benchmark_out={benchmark_dir}", flush=True) print(f"baselines={','.join(baselines)}", flush=True) print(f"isolated={not args.in_process} resume_existing={not args.no_resume_existing}", flush=True) if args.status_only: status = _write_interim_status(output_root, report_path, entries, baselines) print(f"status={output_root / 'run-status.json'} rows={status['rows']} expected_rows={status['expected_rows']}", flush=True) return 0 if not args.skip_benchmark: if args.in_process: run_benchmark( input_paths, benchmark_dir, baselines=baselines, eval_manifest=manifest_path, candidate_model_config=args.candidate_model_config, resume_existing=not args.no_resume_existing, sample_timeout_seconds=args.sample_timeout_seconds, ) else: _run_isolated_benchmark( entries, benchmark_dir, baselines=baselines, manifest_path=manifest_path, candidate_model_config=Path(args.candidate_model_config), resume_existing=not args.no_resume_existing, sample_timeout_seconds=args.sample_timeout_seconds, ) elif not report_path.exists(): raise FileNotFoundError(f"--skip-benchmark requested but report is missing: {report_path}") summary = summarize_benchmark_report( report_path, output_root / "summary", candidate="candidate", baselines=[item for item in baselines if item != "candidate"], ) validation = validate_claim( report_path, output_root / "validation", config_path=args.validation_config, eval_manifest=manifest_path, ) table_analysis = analyze_table_cells( report_path, manifest_path, output_root / "table-cell-analysis", candidate="candidate", baselines=[item for item in baselines if item != "candidate"], ) print(f"report={report_path}", flush=True) print(f"summary={output_root / 'summary' / 'summary.json'} paired={len(summary.paired_metrics)}", flush=True) print(f"validation={validation.claim_status} passed={validation.passed}", flush=True) print( f"table_cell_analysis={output_root / 'table-cell-analysis' / 'table-cell-analysis.json'} " f"systems={len(table_analysis.systems)}", flush=True, ) return 0 if validation.passed else 1 def _run_isolated_benchmark( entries: list[ManifestEntry], benchmark_dir: Path, *, baselines: list[str], manifest_path: Path, candidate_model_config: Path, resume_existing: bool, sample_timeout_seconds: int, ) -> list[BenchmarkResult]: benchmark_dir.mkdir(parents=True, exist_ok=True) status_path = benchmark_dir.parent / "run-status.json" tmp_root = benchmark_dir.parent / "_isolated-tmp" results: list[BenchmarkResult] = _load_existing_report(benchmark_dir / "report.json") completed = {(result.baseline, result.input_path) for result in results} entry_by_path = {str(Path(entry.image_path)): entry for entry in entries} for baseline in baselines: for entry in entries: input_path = Path(entry.image_path) key = (baseline, str(input_path)) if key in completed: continue _write_run_status( status_path, baseline=baseline, sample_id=entry.sample_id, input_path=input_path, rows=len(results), status="running", ) existing_document = benchmark_dir / baseline / input_path.stem / "document.json" if resume_existing and existing_document.exists(): result = _score_existing_document(baseline, input_path, entry, existing_document) else: result = _run_one_isolated( baseline, input_path, manifest_path, benchmark_dir, tmp_root, candidate_model_config=candidate_model_config, entry=entry_by_path[str(input_path)], sample_timeout_seconds=sample_timeout_seconds, ) results.append(result) completed.add(key) _write_reports(results, benchmark_dir) _write_run_status( status_path, baseline=baseline, sample_id=entry.sample_id, input_path=input_path, rows=len(results), status=result.status, error=result.error, ) print( f"progress baseline={baseline} sample={entry.sample_id} status={result.status} rows={len(results)}", flush=True, ) return results def _write_run_status( path: Path, *, baseline: str, sample_id: str, input_path: Path, rows: int, status: str, error: str | None = None, ) -> None: payload = { "updated_at_utc": datetime.now(timezone.utc).isoformat(), "baseline": baseline, "sample_id": sample_id, "input_path": str(input_path), "rows": rows, "status": status, "error": error, } path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _write_interim_status( output_root: Path, report_path: Path, entries: list[ManifestEntry], baselines: list[str], ) -> dict[str, object]: results = _load_existing_report(report_path) completed = {(result.baseline, result.input_path) for result in results} status_counts: dict[str, int] = {} baseline_counts: dict[str, int] = {} per_baseline_progress: dict[str, dict[str, int]] = { baseline: {"expected": len(entries), "completed": 0, "ok": 0, "error": 0, "missing": len(entries)} for baseline in baselines } for result in results: status_counts[result.status] = status_counts.get(result.status, 0) + 1 baseline_counts[result.baseline] = baseline_counts.get(result.baseline, 0) + 1 progress = per_baseline_progress.setdefault( result.baseline, {"expected": len(entries), "completed": 0, "ok": 0, "error": 0, "missing": len(entries)}, ) progress["completed"] += 1 if result.status == "ok": progress["ok"] += 1 elif result.status == "error": progress["error"] += 1 progress["missing"] = max(0, progress["expected"] - progress["completed"]) last_result: dict[str, object] | None = None if results: result = results[-1] last_result = { "baseline": result.baseline, "sample_id": result.sample_id, "input_path": result.input_path, "status": result.status, "error": result.error, } error_results = [result for result in results if result.status == "error"] recent_errors = [ { "baseline": result.baseline, "sample_id": result.sample_id, "input_path": result.input_path, "error": result.error, } for result in error_results[-10:] ] next_item: dict[str, object] | None = None for baseline in baselines: for entry in entries: key = (baseline, str(Path(entry.image_path))) if key in completed: continue next_item = { "baseline": baseline, "sample_id": entry.sample_id, "input_path": str(Path(entry.image_path)), } break if next_item is not None: break final_artifacts = { "summary": (output_root / "summary" / "summary.json").exists(), "validation": (output_root / "validation" / "validation.json").exists(), "table_cell_analysis": (output_root / "table-cell-analysis" / "table-cell-analysis.json").exists(), } validation_status = _load_validation_status(output_root / "validation" / "validation.json") expected_rows = len(entries) * len(baselines) current_phase = str(next_item["baseline"]) if next_item is not None else "finalizing" if not all(final_artifacts.values()) else "complete" evidence_package_ready = len(completed) >= expected_rows and all(final_artifacts.values()) validation_passed = validation_status.get("passed") is True payload: dict[str, object] = { "updated_at_utc": datetime.now(timezone.utc).isoformat(), "status": "complete" if evidence_package_ready else "incomplete", "rows": len(results), "expected_rows": expected_rows, "baselines": baseline_counts, "baseline_progress": per_baseline_progress, "current_phase": current_phase, "row_statuses": status_counts, "last_result": last_result, "error_count": len(error_results), "recent_errors": recent_errors, "next": next_item, "final_artifacts": final_artifacts, "evidence_package_ready": evidence_package_ready, "validation": validation_status, "paper_claim_ready": evidence_package_ready and validation_passed, } (output_root / "run-status.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") _write_interim_status_markdown(output_root / "run-status.md", payload) return payload def _load_validation_status(validation_path: Path) -> dict[str, object]: if not validation_path.exists(): return { "present": False, "claim_status": None, "passed": None, } payload = json.loads(validation_path.read_text(encoding="utf-8")) if not isinstance(payload, dict): raise ValueError(f"validation report must be an object: {validation_path}") return { "present": True, "claim_status": payload.get("claim_status"), "passed": payload.get("passed"), "missing_requirements": payload.get("missing_requirements", []), } def _write_interim_status_markdown(path: Path, payload: dict[str, object]) -> None: baseline_progress = payload.get("baseline_progress") final_artifacts = payload.get("final_artifacts") recent_errors = payload.get("recent_errors") validation_status = payload.get("validation") lines = [ "# Public Table100 Run Status", "", f"Updated: `{payload.get('updated_at_utc')}`", f"Status: `{payload.get('status')}`", f"Current phase: `{payload.get('current_phase')}`", f"Rows: `{payload.get('rows')}` / `{payload.get('expected_rows')}`", f"Evidence package ready: `{'yes' if payload.get('evidence_package_ready') else 'no'}`", f"Paper claim ready: `{'yes' if payload.get('paper_claim_ready') else 'no'}`", "", "## Baseline Progress", "", "| baseline | expected | completed | ok | error | missing |", "| --- | ---: | ---: | ---: | ---: | ---: |", ] if isinstance(baseline_progress, dict): for baseline, progress in sorted(baseline_progress.items()): if not isinstance(progress, dict): continue lines.append( "| {baseline} | {expected} | {completed} | {ok} | {error} | {missing} |".format( baseline=baseline, expected=progress.get("expected", ""), completed=progress.get("completed", ""), ok=progress.get("ok", ""), error=progress.get("error", ""), missing=progress.get("missing", ""), ) ) lines.extend(["", "## Last Result", ""]) last_result = payload.get("last_result") if isinstance(last_result, dict): lines.extend( [ f"- baseline: `{last_result.get('baseline')}`", f"- sample_id: `{last_result.get('sample_id')}`", f"- status: `{last_result.get('status')}`", f"- error: `{last_result.get('error')}`", ] ) else: lines.append("- none") lines.extend(["", "## Recent Errors", ""]) if isinstance(recent_errors, list) and recent_errors: for error in recent_errors: if not isinstance(error, dict): continue lines.append( "- `{baseline}` / `{sample_id}`: {message}".format( baseline=error.get("baseline"), sample_id=error.get("sample_id"), message=error.get("error"), ) ) else: lines.append("- none") lines.extend(["", "## Final Artifacts", ""]) if isinstance(final_artifacts, dict): for name, exists in sorted(final_artifacts.items()): lines.append(f"- {name}: `{'present' if exists else 'missing'}`") else: lines.append("- unknown") lines.extend(["", "## Validation", ""]) if isinstance(validation_status, dict): lines.extend( [ f"- present: `{'yes' if validation_status.get('present') else 'no'}`", f"- claim_status: `{validation_status.get('claim_status')}`", f"- passed: `{validation_status.get('passed')}`", ] ) missing_requirements = validation_status.get("missing_requirements") if isinstance(missing_requirements, list) and missing_requirements: lines.append("- missing requirements:") for requirement in missing_requirements: lines.append(f" - {requirement}") else: lines.append("- unknown") next_item = payload.get("next") lines.extend(["", "## Next", ""]) if isinstance(next_item, dict): lines.extend( [ f"- baseline: `{next_item.get('baseline')}`", f"- sample_id: `{next_item.get('sample_id')}`", f"- input_path: `{next_item.get('input_path')}`", ] ) else: lines.append("- none") path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _load_existing_report(report_path: Path) -> list[BenchmarkResult]: if not report_path.exists(): return [] payload = json.loads(report_path.read_text(encoding="utf-8")) if not isinstance(payload, list): raise ValueError(f"benchmark report must be a list: {report_path}") by_key: dict[tuple[str, str], BenchmarkResult] = {} for row in payload: if not isinstance(row, dict): raise ValueError(f"benchmark report row must be an object: {report_path}") result = BenchmarkResult( baseline=str(row.get("baseline") or ""), input_path=str(row.get("input_path") or ""), status=str(row.get("status") or "error"), latency_seconds=float(row.get("latency_seconds") or 0.0), sample_id=str(row.get("sample_id") or "") or None, slices=[str(item) for item in row.get("slices", [])] if isinstance(row.get("slices"), list) else [], metrics={str(key): float(value) for key, value in dict(row.get("metrics") or {}).items()}, error=str(row.get("error") or "") or None, ) by_key[(result.baseline, result.input_path)] = result return list(by_key.values()) def _score_existing_document(baseline: str, input_path: Path, entry: ManifestEntry, document_path: Path) -> BenchmarkResult: payload = json.loads(document_path.read_text(encoding="utf-8")) if not isinstance(payload, dict): raise ValueError(f"document JSON must be an object: {document_path}") document = Document.from_dict(payload) reference_path = entry.metadata.get("reference_path") explicit_reference_path = str(reference_path) if isinstance(reference_path, str) else None metrics = score_document(document, document.text, load_reference(input_path, None, explicit_path=explicit_reference_path)) metrics.update(_quality_metrics(document)) return BenchmarkResult( baseline=baseline, input_path=str(input_path), status="ok", latency_seconds=0.0, sample_id=entry.sample_id, slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [], metrics=metrics, ) def _run_one_isolated( baseline: str, input_path: Path, manifest_path: Path, benchmark_dir: Path, tmp_root: Path, *, candidate_model_config: Path, entry: ManifestEntry, sample_timeout_seconds: int, ) -> BenchmarkResult: tmp_dir = tmp_root / baseline / input_path.stem if tmp_dir.exists(): shutil.rmtree(tmp_dir) tmp_dir.mkdir(parents=True) command = [ sys.executable, "-m", "ocrtech.cli", "benchmark", str(input_path), "--out", str(tmp_dir), "--baselines", baseline, "--eval-manifest", str(manifest_path), ] if baseline == "candidate": command.extend(["--candidate-model-config", str(candidate_model_config)]) try: completed = subprocess.run( command, check=False, capture_output=True, text=True, timeout=sample_timeout_seconds, ) except subprocess.TimeoutExpired as exc: return BenchmarkResult( baseline=baseline, input_path=str(input_path), status="error", latency_seconds=float(sample_timeout_seconds), sample_id=entry.sample_id, slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [], error=f"TimeoutExpired: timed out after {sample_timeout_seconds}s", ) tmp_report = tmp_dir / "report.json" if not tmp_report.exists(): stderr_tail = (completed.stderr or completed.stdout or "").strip().splitlines()[-5:] return BenchmarkResult( baseline=baseline, input_path=str(input_path), status="error", latency_seconds=0.0, sample_id=entry.sample_id, slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [], error=f"isolated benchmark exited {completed.returncode}: {' | '.join(stderr_tail)}", ) rows = json.loads(tmp_report.read_text(encoding="utf-8")) if not isinstance(rows, list) or len(rows) != 1 or not isinstance(rows[0], dict): raise ValueError(f"isolated benchmark report must contain exactly one row: {tmp_report}") output_source = tmp_dir / baseline / input_path.stem output_target = benchmark_dir / baseline / input_path.stem if output_source.exists(): if output_target.exists(): shutil.rmtree(output_target) shutil.copytree(output_source, output_target) row = rows[0] return BenchmarkResult( baseline=str(row.get("baseline") or baseline), input_path=str(row.get("input_path") or input_path), status=str(row.get("status") or "error"), latency_seconds=float(row.get("latency_seconds") or 0.0), sample_id=str(row.get("sample_id") or entry.sample_id) or None, slices=[str(item) for item in row.get("slices", [])] if isinstance(row.get("slices"), list) else [], metrics={str(key): float(value) for key, value in dict(row.get("metrics") or {}).items()}, error=str(row.get("error") or "") or None, ) if __name__ == "__main__": raise SystemExit(main())