eval: add --retry-errors to re-predict only the entries that failed

A quota outage left 44 of 361 cached SROIE documents with no extraction.
Recovering them with --overwrite would have re-run all 361 and replaced the
317 successful predictions with fresh model output, invalidating every
before/after rule comparison built on them -- S2 and the monetary tolerance
change both depend on the predictions being frozen while only the rules vary.

--retry-errors selects exactly the entries whose error field is set and skips
every other cached id, so a successful prediction is never re-run, re-billed,
or rewritten. Two guards, because both failure modes here spend quota
silently: combining it with --overwrite raises rather than letting one win,
and using it against an empty cache raises rather than falling through to a
full predict.

Note this leaves a cache that is mixed by construction: the backfilled 44
carry confidence and validation computed under current rules, the original
317 under the rules in force when they ran. That is only a disk-level
inconsistency. Scoring with --revalidate recomputes both from the cached
predicted document, which is model output and therefore rule-independent, so
every entry is judged under one rule set regardless of when it was predicted.
The drift check reports the disk-level disagreement rather than hiding it.

eval/cache.py

@@ -90,6 +90,25 @@ def existing_ids(cache_base: Path, dataset: str) -> set[str]:
     return {str(entry["id"]) for entry in read_entries(cache_base, dataset)}
 
 
+def errored_ids(cache_base: Path, dataset: str) -> set[str]:
+    """Return the ids of cached entries whose pipeline run recorded an error.
+
+    These are documents that produced no extraction -- a quota outage, a
+    timeout, an unreadable file -- as distinct from documents the model read
+    and the rules then rejected. Only these are worth re-running: a successful
+    prediction must stay frozen so that before/after rule comparisons remain
+    attributable to the rule change.
+
+    Args:
+        cache_base: Root cache directory.
+        dataset: Dataset name (subdirectory).
+
+    Returns:
+        The set of example ids with a non-empty ``error`` field.
+    """
+    return {str(e["id"]) for e in read_entries(cache_base, dataset) if e.get("error")}
+
+
 def report_from_dict(validation: dict[str, Any]) -> ValidationReport:
     """Reconstruct a :class:`ValidationReport` from its cached dict form.
 

eval/predict.py

@@ -25,7 +25,7 @@ from doc_agent.backends.base import create_backend
 from doc_agent.config import Settings, load_config
 from doc_agent.core import process_document
 
-from eval.cache import DEFAULT_CACHE_BASE, existing_ids, write_entry
+from eval.cache import DEFAULT_CACHE_BASE, errored_ids, existing_ids, write_entry
 from eval.datasets import WIRED_DATASETS, get_adapter
 
 logger = logging.getLogger(__name__)
@@ -119,6 +119,7 @@ def run_predict(
     settings: Settings | None = None,
     cache_base: Path = DEFAULT_CACHE_BASE,
     overwrite: bool = False,
+    retry_errors: bool = False,
 ) -> PredictStats:
     """Run the pipeline over a dataset slice and cache each result.
 
@@ -131,13 +132,23 @@ def run_predict(
         cache_base: Root cache directory. Defaults to ``eval/cache``.
         overwrite: Re-process and overwrite examples already cached. Defaults to
             ``False`` so re-runs resume without re-billing.
+        retry_errors: Re-process *only* cached entries that recorded an error,
+            leaving every successful prediction byte-identical. Use after an
+            outage. Mutually exclusive with ``overwrite``.
 
     Returns:
         A :class:`PredictStats` summary of the run.
 
     Raises:
-        ValueError: If ``dataset`` is not wired for the predict phase.
+        ValueError: If ``dataset`` is not wired for the predict phase, if both
+            ``overwrite`` and ``retry_errors`` are set, or if ``retry_errors``
+            is set with no cache to retry from.
     """
+    if overwrite and retry_errors:
+        raise ValueError(
+            "--overwrite and --retry-errors are mutually exclusive: the first "
+            "re-runs every document, the second only the failed ones. Pick one."
+        )
     if dataset not in WIRED_DATASETS:
         wired = ", ".join(sorted(WIRED_DATASETS))
         raise ValueError(
@@ -146,9 +157,28 @@ def run_predict(
         )
 
     adapter = get_adapter(dataset)
+
+    if retry_errors:
+        cached = existing_ids(cache_base, dataset)
+        if not cached:
+            raise ValueError(
+                f"--retry-errors needs an existing cache for dataset {dataset!r} "
+                f"under {cache_base}, but none was found. Run a normal predict first."
+            )
+        retry = errored_ids(cache_base, dataset)
+        # Skip everything already cached that did NOT error, so successful
+        # predictions are never re-run, re-billed, or rewritten.
+        already = cached - retry
+        logger.info(
+            "eval-predict: retry-errors mode -- %d errored of %d cached will be re-run",
+            len(retry),
+            len(cached),
+        )
+    else:
+        already = set() if overwrite else existing_ids(cache_base, dataset)
+
     settings = settings or load_config()
     backend = create_backend(settings)
-    already = set() if overwrite else existing_ids(cache_base, dataset)
 
     processed = skipped = accepted = review = errors = failed = 0
     logger.info("eval-predict: dataset=%s limit=%s backend=%s", dataset, limit, backend.name)

eval/run_eval.py

@@ -54,6 +54,16 @@ def _build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Re-process and overwrite already-cached examples.",
     )
+    predict.add_argument(
+        "--retry-errors",
+        action="store_true",
+        help=(
+            "Re-predict ONLY cached entries whose 'error' field is set, leaving "
+            "every successful prediction byte-identical. Use after a quota or "
+            "network outage. Unlike --overwrite this never re-runs a document "
+            "that already succeeded, so frozen predictions stay comparable."
+        ),
+    )
 
     score = subparsers.add_parser("score", help="Compute metrics + sweep from the cache (offline).")
     _add_common(score)
@@ -103,6 +113,7 @@ def main(argv: list[str] | None = None) -> int:
             args.limit,
             cache_base=args.cache_base,
             overwrite=args.overwrite,
+            retry_errors=args.retry_errors,
         )
         print(
             f"\nPredict complete for {stats.dataset}: "

tests/test_eval.py

@@ -13,7 +13,7 @@ from typing import Any
 
 import pytest
 
-from eval.cache import read_entries, report_from_dict, write_entry
+from eval.cache import errored_ids, read_entries, report_from_dict, write_entry
 from eval.metrics import (
     THRESHOLDS,
     compute_field_metrics,
@@ -22,6 +22,7 @@ from eval.metrics import (
     sweep_thresholds,
 )
 from eval.normalize import is_present, normalize, values_match
+from eval.predict import run_predict
 from eval.score import build_report
 
 
@@ -315,3 +316,49 @@ def test_build_report_raises_without_cache(tmp_path: Path) -> None:
     """Scoring a dataset with no cache raises a clear error."""
     with pytest.raises(FileNotFoundError):
         build_report("missing", cache_base=tmp_path)
+
+
+# ---------------------------------------------------------------------------
+# Targeted retry of failed predictions (--retry-errors)
+# ---------------------------------------------------------------------------
+
+
+def _cached(example_id: str, *, error: str | None) -> dict[str, Any]:
+    entry = _entry(
+        example_id,
+        predicted={"total": 1.0},
+        gold={"total": "1.00"},
+        confidence=0.5,
+    )
+    entry["error"] = error
+    return entry
+
+
+def test_errored_ids_selects_only_failed_entries(tmp_path: Path) -> None:
+    """The retry set is exactly the entries that produced no extraction.
+
+    A document the model read and the rules then rejected is a result, not a
+    failure, and must never be re-run -- re-running it would replace a frozen
+    prediction and break any before/after rule comparison built on it.
+    """
+    for name, error in (("ok1", None), ("ok2", None), ("dead", "429 RESOURCE_EXHAUSTED")):
+        write_entry(tmp_path, "d", _cached(name, error=error))
+
+    assert errored_ids(tmp_path, "d") == {"dead"}
+
+
+def test_errored_ids_is_empty_for_a_clean_cache(tmp_path: Path) -> None:
+    write_entry(tmp_path, "d", _cached("ok", error=None))
+    assert errored_ids(tmp_path, "d") == set()
+
+
+def test_overwrite_and_retry_errors_are_mutually_exclusive(tmp_path: Path) -> None:
+    """Silently letting one win would re-run 361 documents when 44 were meant."""
+    with pytest.raises(ValueError, match="mutually exclusive"):
+        run_predict("sroie", 1, cache_base=tmp_path, overwrite=True, retry_errors=True)
+
+
+def test_retry_errors_refuses_an_empty_cache(tmp_path: Path) -> None:
+    """Falling through to a full predict here would spend quota unasked."""
+    with pytest.raises(ValueError, match="needs an existing cache"):
+        run_predict("sroie", 1, cache_base=tmp_path, retry_errors=True)