scripts/scavenger_hunt/validate_dataset.py

"""
validate_dataset.py — scavenger_hunt
Re-runs validate_example() on every record in a dataset.json and reports
pass/fail counts plus failure-category breakdown. Also reports how many
examples were skipped during generation (dataset_errors.jsonl).

Usage:
  python validate_dataset.py [path/to/dataset.json]
  (defaults to app/data/scavenger_hunt/dataset.json)
"""
import json
import os
import sys
from collections import Counter

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from validator import validate_example

DEFAULT_PATH = os.path.normpath(os.path.join(
    HERE, "..", "..", "app", "data", "scavenger_hunt", "dataset.json"))


def main():
    path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_PATH
    if not os.path.exists(path):
        print(f"File not found: {path}")
        sys.exit(1)

    data = json.load(open(path, encoding="utf-8"))
    passed = failed = 0
    failure_reasons = Counter()
    failed_ids = []

    for ex in data:
        ok, errors = validate_example(ex)
        if ok:
            passed += 1
        else:
            failed += 1
            failed_ids.append(ex.get("id", "?"))
            for e in errors:
                failure_reasons[e.split(":")[0]] += 1

    print(f"\n── Validation report: {path} ──────────────────")
    print(f"  Total   : {len(data)}")
    print(f"  Passed  : {passed}")
    print(f"  Failed  : {failed}")
    if failed:
        print("\n  Failure categories:")
        for reason, count in failure_reasons.most_common():
            print(f"    {reason:10s}: {count}")
        print("\n  Failed example ids:", failed_ids)

    errors_path = path.replace(".json", "_errors.jsonl")
    if os.path.exists(errors_path):
        with open(errors_path, encoding="utf-8") as f:
            n_skipped = sum(1 for _ in f)
        print(f"\n  Skipped during generation (logged in {errors_path}): {n_skipped}")
    else:
        print(f"\n  No {os.path.basename(errors_path)} found — no examples were skipped during generation.")


if __name__ == "__main__":
    main()