ocr-projects-archive / ocr-tech-mps-integration /scripts /run_public_table100_claim.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
22.7 kB
#!/usr/bin/env python3
"""Run the 100-page public OmniDocBench table claim benchmark."""
from __future__ import annotations
import argparse
import json
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
from ocrtech.benchmark import BenchmarkResult, _quality_metrics, _write_reports, run_benchmark, summarize_benchmark_report
from ocrtech.manifest import ManifestEntry, load_manifest
from ocrtech.references import load_reference, score_document
from ocrtech.schemas import Document
from ocrtech.table_analysis import analyze_table_cells
from ocrtech.validation import validate_claim
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Run public OmniDocBench table100 candidate-vs-Surya evidence.")
parser.add_argument(
"--manifest",
default="data/public/omnidocbench-table100/manifests/omnidocbench-eval.jsonl",
help="audited OmniDocBench eval manifest",
)
parser.add_argument(
"--candidate-model-config",
default="outputs/figure-caption-table-lexicon-normalize-width-guard-v4/model-card.json",
help="candidate model card",
)
parser.add_argument("--out", default="outputs/experiment-public-omnidocbench-table100-width-guard-v4")
parser.add_argument("--validation-config", default="configs/validation-public-omnidocbench-surya.json")
parser.add_argument("--baselines", default="candidate,surya")
parser.add_argument("--skip-benchmark", action="store_true", help="reuse an existing benchmark/report.json")
parser.add_argument("--no-resume-existing", action="store_true", help="ignore existing document.json outputs")
parser.add_argument("--sample-timeout-seconds", type=int, default=300)
parser.add_argument("--in-process", action="store_true", help="run the benchmark in one Python process instead of isolated page subprocesses")
parser.add_argument(
"--status-only",
action="store_true",
help="write run-status.json and run-status.md from the current checkpoint without running benchmark, summary, validation, or analysis",
)
return parser
def main() -> int:
args = build_parser().parse_args()
manifest_path = Path(args.manifest)
output_root = Path(args.out)
benchmark_dir = output_root / "benchmark"
report_path = benchmark_dir / "report.json"
baselines = [item.strip() for item in args.baselines.split(",") if item.strip()]
if not baselines:
raise ValueError("--baselines must include at least one baseline")
entries = load_manifest(manifest_path)
input_paths = [entry.image_path for entry in entries]
print(f"manifest={manifest_path} samples={len(entries)}", flush=True)
print(f"benchmark_out={benchmark_dir}", flush=True)
print(f"baselines={','.join(baselines)}", flush=True)
print(f"isolated={not args.in_process} resume_existing={not args.no_resume_existing}", flush=True)
if args.status_only:
status = _write_interim_status(output_root, report_path, entries, baselines)
print(f"status={output_root / 'run-status.json'} rows={status['rows']} expected_rows={status['expected_rows']}", flush=True)
return 0
if not args.skip_benchmark:
if args.in_process:
run_benchmark(
input_paths,
benchmark_dir,
baselines=baselines,
eval_manifest=manifest_path,
candidate_model_config=args.candidate_model_config,
resume_existing=not args.no_resume_existing,
sample_timeout_seconds=args.sample_timeout_seconds,
)
else:
_run_isolated_benchmark(
entries,
benchmark_dir,
baselines=baselines,
manifest_path=manifest_path,
candidate_model_config=Path(args.candidate_model_config),
resume_existing=not args.no_resume_existing,
sample_timeout_seconds=args.sample_timeout_seconds,
)
elif not report_path.exists():
raise FileNotFoundError(f"--skip-benchmark requested but report is missing: {report_path}")
summary = summarize_benchmark_report(
report_path,
output_root / "summary",
candidate="candidate",
baselines=[item for item in baselines if item != "candidate"],
)
validation = validate_claim(
report_path,
output_root / "validation",
config_path=args.validation_config,
eval_manifest=manifest_path,
)
table_analysis = analyze_table_cells(
report_path,
manifest_path,
output_root / "table-cell-analysis",
candidate="candidate",
baselines=[item for item in baselines if item != "candidate"],
)
print(f"report={report_path}", flush=True)
print(f"summary={output_root / 'summary' / 'summary.json'} paired={len(summary.paired_metrics)}", flush=True)
print(f"validation={validation.claim_status} passed={validation.passed}", flush=True)
print(
f"table_cell_analysis={output_root / 'table-cell-analysis' / 'table-cell-analysis.json'} "
f"systems={len(table_analysis.systems)}",
flush=True,
)
return 0 if validation.passed else 1
def _run_isolated_benchmark(
entries: list[ManifestEntry],
benchmark_dir: Path,
*,
baselines: list[str],
manifest_path: Path,
candidate_model_config: Path,
resume_existing: bool,
sample_timeout_seconds: int,
) -> list[BenchmarkResult]:
benchmark_dir.mkdir(parents=True, exist_ok=True)
status_path = benchmark_dir.parent / "run-status.json"
tmp_root = benchmark_dir.parent / "_isolated-tmp"
results: list[BenchmarkResult] = _load_existing_report(benchmark_dir / "report.json")
completed = {(result.baseline, result.input_path) for result in results}
entry_by_path = {str(Path(entry.image_path)): entry for entry in entries}
for baseline in baselines:
for entry in entries:
input_path = Path(entry.image_path)
key = (baseline, str(input_path))
if key in completed:
continue
_write_run_status(
status_path,
baseline=baseline,
sample_id=entry.sample_id,
input_path=input_path,
rows=len(results),
status="running",
)
existing_document = benchmark_dir / baseline / input_path.stem / "document.json"
if resume_existing and existing_document.exists():
result = _score_existing_document(baseline, input_path, entry, existing_document)
else:
result = _run_one_isolated(
baseline,
input_path,
manifest_path,
benchmark_dir,
tmp_root,
candidate_model_config=candidate_model_config,
entry=entry_by_path[str(input_path)],
sample_timeout_seconds=sample_timeout_seconds,
)
results.append(result)
completed.add(key)
_write_reports(results, benchmark_dir)
_write_run_status(
status_path,
baseline=baseline,
sample_id=entry.sample_id,
input_path=input_path,
rows=len(results),
status=result.status,
error=result.error,
)
print(
f"progress baseline={baseline} sample={entry.sample_id} status={result.status} rows={len(results)}",
flush=True,
)
return results
def _write_run_status(
path: Path,
*,
baseline: str,
sample_id: str,
input_path: Path,
rows: int,
status: str,
error: str | None = None,
) -> None:
payload = {
"updated_at_utc": datetime.now(timezone.utc).isoformat(),
"baseline": baseline,
"sample_id": sample_id,
"input_path": str(input_path),
"rows": rows,
"status": status,
"error": error,
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _write_interim_status(
output_root: Path,
report_path: Path,
entries: list[ManifestEntry],
baselines: list[str],
) -> dict[str, object]:
results = _load_existing_report(report_path)
completed = {(result.baseline, result.input_path) for result in results}
status_counts: dict[str, int] = {}
baseline_counts: dict[str, int] = {}
per_baseline_progress: dict[str, dict[str, int]] = {
baseline: {"expected": len(entries), "completed": 0, "ok": 0, "error": 0, "missing": len(entries)}
for baseline in baselines
}
for result in results:
status_counts[result.status] = status_counts.get(result.status, 0) + 1
baseline_counts[result.baseline] = baseline_counts.get(result.baseline, 0) + 1
progress = per_baseline_progress.setdefault(
result.baseline,
{"expected": len(entries), "completed": 0, "ok": 0, "error": 0, "missing": len(entries)},
)
progress["completed"] += 1
if result.status == "ok":
progress["ok"] += 1
elif result.status == "error":
progress["error"] += 1
progress["missing"] = max(0, progress["expected"] - progress["completed"])
last_result: dict[str, object] | None = None
if results:
result = results[-1]
last_result = {
"baseline": result.baseline,
"sample_id": result.sample_id,
"input_path": result.input_path,
"status": result.status,
"error": result.error,
}
error_results = [result for result in results if result.status == "error"]
recent_errors = [
{
"baseline": result.baseline,
"sample_id": result.sample_id,
"input_path": result.input_path,
"error": result.error,
}
for result in error_results[-10:]
]
next_item: dict[str, object] | None = None
for baseline in baselines:
for entry in entries:
key = (baseline, str(Path(entry.image_path)))
if key in completed:
continue
next_item = {
"baseline": baseline,
"sample_id": entry.sample_id,
"input_path": str(Path(entry.image_path)),
}
break
if next_item is not None:
break
final_artifacts = {
"summary": (output_root / "summary" / "summary.json").exists(),
"validation": (output_root / "validation" / "validation.json").exists(),
"table_cell_analysis": (output_root / "table-cell-analysis" / "table-cell-analysis.json").exists(),
}
validation_status = _load_validation_status(output_root / "validation" / "validation.json")
expected_rows = len(entries) * len(baselines)
current_phase = str(next_item["baseline"]) if next_item is not None else "finalizing" if not all(final_artifacts.values()) else "complete"
evidence_package_ready = len(completed) >= expected_rows and all(final_artifacts.values())
validation_passed = validation_status.get("passed") is True
payload: dict[str, object] = {
"updated_at_utc": datetime.now(timezone.utc).isoformat(),
"status": "complete" if evidence_package_ready else "incomplete",
"rows": len(results),
"expected_rows": expected_rows,
"baselines": baseline_counts,
"baseline_progress": per_baseline_progress,
"current_phase": current_phase,
"row_statuses": status_counts,
"last_result": last_result,
"error_count": len(error_results),
"recent_errors": recent_errors,
"next": next_item,
"final_artifacts": final_artifacts,
"evidence_package_ready": evidence_package_ready,
"validation": validation_status,
"paper_claim_ready": evidence_package_ready and validation_passed,
}
(output_root / "run-status.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_interim_status_markdown(output_root / "run-status.md", payload)
return payload
def _load_validation_status(validation_path: Path) -> dict[str, object]:
if not validation_path.exists():
return {
"present": False,
"claim_status": None,
"passed": None,
}
payload = json.loads(validation_path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError(f"validation report must be an object: {validation_path}")
return {
"present": True,
"claim_status": payload.get("claim_status"),
"passed": payload.get("passed"),
"missing_requirements": payload.get("missing_requirements", []),
}
def _write_interim_status_markdown(path: Path, payload: dict[str, object]) -> None:
baseline_progress = payload.get("baseline_progress")
final_artifacts = payload.get("final_artifacts")
recent_errors = payload.get("recent_errors")
validation_status = payload.get("validation")
lines = [
"# Public Table100 Run Status",
"",
f"Updated: `{payload.get('updated_at_utc')}`",
f"Status: `{payload.get('status')}`",
f"Current phase: `{payload.get('current_phase')}`",
f"Rows: `{payload.get('rows')}` / `{payload.get('expected_rows')}`",
f"Evidence package ready: `{'yes' if payload.get('evidence_package_ready') else 'no'}`",
f"Paper claim ready: `{'yes' if payload.get('paper_claim_ready') else 'no'}`",
"",
"## Baseline Progress",
"",
"| baseline | expected | completed | ok | error | missing |",
"| --- | ---: | ---: | ---: | ---: | ---: |",
]
if isinstance(baseline_progress, dict):
for baseline, progress in sorted(baseline_progress.items()):
if not isinstance(progress, dict):
continue
lines.append(
"| {baseline} | {expected} | {completed} | {ok} | {error} | {missing} |".format(
baseline=baseline,
expected=progress.get("expected", ""),
completed=progress.get("completed", ""),
ok=progress.get("ok", ""),
error=progress.get("error", ""),
missing=progress.get("missing", ""),
)
)
lines.extend(["", "## Last Result", ""])
last_result = payload.get("last_result")
if isinstance(last_result, dict):
lines.extend(
[
f"- baseline: `{last_result.get('baseline')}`",
f"- sample_id: `{last_result.get('sample_id')}`",
f"- status: `{last_result.get('status')}`",
f"- error: `{last_result.get('error')}`",
]
)
else:
lines.append("- none")
lines.extend(["", "## Recent Errors", ""])
if isinstance(recent_errors, list) and recent_errors:
for error in recent_errors:
if not isinstance(error, dict):
continue
lines.append(
"- `{baseline}` / `{sample_id}`: {message}".format(
baseline=error.get("baseline"),
sample_id=error.get("sample_id"),
message=error.get("error"),
)
)
else:
lines.append("- none")
lines.extend(["", "## Final Artifacts", ""])
if isinstance(final_artifacts, dict):
for name, exists in sorted(final_artifacts.items()):
lines.append(f"- {name}: `{'present' if exists else 'missing'}`")
else:
lines.append("- unknown")
lines.extend(["", "## Validation", ""])
if isinstance(validation_status, dict):
lines.extend(
[
f"- present: `{'yes' if validation_status.get('present') else 'no'}`",
f"- claim_status: `{validation_status.get('claim_status')}`",
f"- passed: `{validation_status.get('passed')}`",
]
)
missing_requirements = validation_status.get("missing_requirements")
if isinstance(missing_requirements, list) and missing_requirements:
lines.append("- missing requirements:")
for requirement in missing_requirements:
lines.append(f" - {requirement}")
else:
lines.append("- unknown")
next_item = payload.get("next")
lines.extend(["", "## Next", ""])
if isinstance(next_item, dict):
lines.extend(
[
f"- baseline: `{next_item.get('baseline')}`",
f"- sample_id: `{next_item.get('sample_id')}`",
f"- input_path: `{next_item.get('input_path')}`",
]
)
else:
lines.append("- none")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _load_existing_report(report_path: Path) -> list[BenchmarkResult]:
if not report_path.exists():
return []
payload = json.loads(report_path.read_text(encoding="utf-8"))
if not isinstance(payload, list):
raise ValueError(f"benchmark report must be a list: {report_path}")
by_key: dict[tuple[str, str], BenchmarkResult] = {}
for row in payload:
if not isinstance(row, dict):
raise ValueError(f"benchmark report row must be an object: {report_path}")
result = BenchmarkResult(
baseline=str(row.get("baseline") or ""),
input_path=str(row.get("input_path") or ""),
status=str(row.get("status") or "error"),
latency_seconds=float(row.get("latency_seconds") or 0.0),
sample_id=str(row.get("sample_id") or "") or None,
slices=[str(item) for item in row.get("slices", [])] if isinstance(row.get("slices"), list) else [],
metrics={str(key): float(value) for key, value in dict(row.get("metrics") or {}).items()},
error=str(row.get("error") or "") or None,
)
by_key[(result.baseline, result.input_path)] = result
return list(by_key.values())
def _score_existing_document(baseline: str, input_path: Path, entry: ManifestEntry, document_path: Path) -> BenchmarkResult:
payload = json.loads(document_path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError(f"document JSON must be an object: {document_path}")
document = Document.from_dict(payload)
reference_path = entry.metadata.get("reference_path")
explicit_reference_path = str(reference_path) if isinstance(reference_path, str) else None
metrics = score_document(document, document.text, load_reference(input_path, None, explicit_path=explicit_reference_path))
metrics.update(_quality_metrics(document))
return BenchmarkResult(
baseline=baseline,
input_path=str(input_path),
status="ok",
latency_seconds=0.0,
sample_id=entry.sample_id,
slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [],
metrics=metrics,
)
def _run_one_isolated(
baseline: str,
input_path: Path,
manifest_path: Path,
benchmark_dir: Path,
tmp_root: Path,
*,
candidate_model_config: Path,
entry: ManifestEntry,
sample_timeout_seconds: int,
) -> BenchmarkResult:
tmp_dir = tmp_root / baseline / input_path.stem
if tmp_dir.exists():
shutil.rmtree(tmp_dir)
tmp_dir.mkdir(parents=True)
command = [
sys.executable,
"-m",
"ocrtech.cli",
"benchmark",
str(input_path),
"--out",
str(tmp_dir),
"--baselines",
baseline,
"--eval-manifest",
str(manifest_path),
]
if baseline == "candidate":
command.extend(["--candidate-model-config", str(candidate_model_config)])
try:
completed = subprocess.run(
command,
check=False,
capture_output=True,
text=True,
timeout=sample_timeout_seconds,
)
except subprocess.TimeoutExpired as exc:
return BenchmarkResult(
baseline=baseline,
input_path=str(input_path),
status="error",
latency_seconds=float(sample_timeout_seconds),
sample_id=entry.sample_id,
slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [],
error=f"TimeoutExpired: timed out after {sample_timeout_seconds}s",
)
tmp_report = tmp_dir / "report.json"
if not tmp_report.exists():
stderr_tail = (completed.stderr or completed.stdout or "").strip().splitlines()[-5:]
return BenchmarkResult(
baseline=baseline,
input_path=str(input_path),
status="error",
latency_seconds=0.0,
sample_id=entry.sample_id,
slices=[str(item) for item in entry.metadata.get("slices", [])] if isinstance(entry.metadata.get("slices"), list) else [],
error=f"isolated benchmark exited {completed.returncode}: {' | '.join(stderr_tail)}",
)
rows = json.loads(tmp_report.read_text(encoding="utf-8"))
if not isinstance(rows, list) or len(rows) != 1 or not isinstance(rows[0], dict):
raise ValueError(f"isolated benchmark report must contain exactly one row: {tmp_report}")
output_source = tmp_dir / baseline / input_path.stem
output_target = benchmark_dir / baseline / input_path.stem
if output_source.exists():
if output_target.exists():
shutil.rmtree(output_target)
shutil.copytree(output_source, output_target)
row = rows[0]
return BenchmarkResult(
baseline=str(row.get("baseline") or baseline),
input_path=str(row.get("input_path") or input_path),
status=str(row.get("status") or "error"),
latency_seconds=float(row.get("latency_seconds") or 0.0),
sample_id=str(row.get("sample_id") or entry.sample_id) or None,
slices=[str(item) for item in row.get("slices", [])] if isinstance(row.get("slices"), list) else [],
metrics={str(key): float(value) for key, value in dict(row.get("metrics") or {}).items()},
error=str(row.get("error") or "") or None,
)
if __name__ == "__main__":
raise SystemExit(main())