run_eval.py

"""
Standalone evaluation runner for the contract drafting system.
Runs on CPU; no GPU required for template-based generation.
"""

import json
import sys

from clause_retriever import ClauseRetriever, build_retriever_from_hf_datasets
from drafting_engine import ContractDraftingEngine, DraftingContext
from eval_runner import EvalRunner, GOLD_TASKS


def main():
    print("Initializing clause retriever...")
    retriever = ClauseRetriever(use_bm25=True, use_embeddings=False)
    # Load small seed clause library
    try:
        from datasets import load_dataset
        ds = load_dataset("asapworks/Contract_Clause_SampleDataset", split="train")
        for row in ds:
            retriever.add_clauses([{
                "clause_text": row["clause_text"],
                "clause_type": row.get("clause_type", "unknown"),
                "source": row.get("file", "seed"),
            }])
        print(f"Loaded {len(retriever.corpus)} seed clauses.")
    except Exception as e:
        print(f"Warning: could not load seed clauses: {e}")

    engine = ContractDraftingEngine(retriever=retriever)
    runner = EvalRunner(engine)

    print("Running gold task suite...")
    results = runner.run_suite(GOLD_TASKS)
    report = runner.report(results)
    print(report)

    with open("eval_report.md", "w") as f:
        f.write(report)

    # Save detailed JSON
    detailed = []
    for r in results:
        detailed.append({
            "task_id": r.task_id,
            "contract_type": r.contract_type,
            "total_score": r.total_score,
            "scores": r.scores,
        })
    with open("eval_results.json", "w") as f:
        json.dump(detailed, f, indent=2)

    print("\nSaved eval_report.md and eval_results.json")


if __name__ == "__main__":
    main()