File size: 1,107 Bytes
f8a5189 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env python
"""Validate an eval set against its docs folder; exit 1 if anything is inconsistent.
Usage:
python scripts/validate_eval_set.py # data/eval_v2
python scripts/validate_eval_set.py EVAL.jsonl DOCS_DIR
"""
from __future__ import annotations
import sys
from pathlib import Path
ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT))
from lumen_rag.eval.validate import validate_eval_set
def main() -> int:
if len(sys.argv) == 3:
eval_path, docs_dir = Path(sys.argv[1]), Path(sys.argv[2])
else:
eval_path = ROOT / "data" / "eval_v2" / "eval.jsonl"
docs_dir = ROOT / "data" / "eval_v2" / "docs"
errors, stats = validate_eval_set(eval_path, docs_dir)
print(f"docs={stats['n_docs']} cases={stats['n_cases']} categories={stats['categories']}")
for err in errors:
print(" ERROR", err, file=sys.stderr)
if errors:
print(f"{len(errors)} problem(s) found", file=sys.stderr)
return 1
print("eval set is consistent")
return 0
if __name__ == "__main__":
sys.exit(main())
|