run_full.py

"""
Self-contained contract drafting system + evaluation.
Downloads all code from the repo and runs everything.
"""

import os
import sys
import json

# Install deps
os.system("pip install -q datasets rank-bm25 sentence-transformers numpy")

# Download source files from the Hub
from huggingface_hub import hf_hub_download

repo = "narcolepticchicken/contract-drafting-assistant"
for fname in ["playbook.py", "clause_retriever.py", "drafting_engine.py", "eval_runner.py"]:
    path = hf_hub_download(repo_id=repo, filename=fname)
    os.makedirs("/app", exist_ok=True)
    os.system(f"cp {path} /app/{fname}")

sys.path.insert(0, "/app")

from clause_retriever import ClauseRetriever
from drafting_engine import ContractDraftingEngine, DraftingContext
from eval_runner import EvalRunner, GOLD_TASKS

print("=" * 60)
print("CONTRACT DRAFTING ASSISTANT - FULL EVALUATION RUN")
print("=" * 60)

# Build retriever with seed clauses
print("\n[1] Building clause retriever...")
retriever = ClauseRetriever(use_bm25=True, use_embeddings=False)
try:
    from datasets import load_dataset
    ds = load_dataset("asapworks/Contract_Clause_SampleDataset", split="train")
    for row in ds:
        retriever.add_clauses([{
            "clause_text": row["clause_text"],
            "clause_type": row.get("clause_type", "unknown"),
            "source": row.get("file", "seed"),
        }])
    print(f"  Loaded {len(retriever.corpus)} seed clauses from asapworks/Contract_Clause_SampleDataset")
except Exception as e:
    print(f"  Warning: could not load seed clauses: {e}")

# Also try loading ContractNLI for additional context
try:
    ds = load_dataset("kiddothe2b/contract-nli", "contractnli_a", split="train")
    for row in ds:
        retriever.add_clauses([{
            "clause_text": row["premise"],
            "clause_type": "nda_clause",
            "source": "contract-nli",
        }])
    print(f"  Loaded contract-nli premises")
except Exception as e:
    print(f"  Warning: could not load contract-nli: {e}")

# Build engine
print("\n[2] Initializing drafting engine...")
engine = ContractDraftingEngine(retriever=retriever)

# Run eval suite
print("\n[3] Running gold task evaluation suite...")
runner = EvalRunner(engine)
results = runner.run_suite(GOLD_TASKS)

# Print report
report = runner.report(results)
print(report)

# Save report
with open("/app/eval_report.md", "w") as f:
    f.write(report)

# Save detailed JSON
detailed = []
for r in results:
    detailed.append({
        "task_id": r.task_id,
        "contract_type": r.contract_type,
        "total_score": r.total_score,
        "scores": r.scores,
    })
with open("/app/eval_results.json", "w") as f:
    json.dump(detailed, f, indent=2)

# Generate sample drafted agreements
print("\n[4] Generating sample drafted agreements...")
samples = [
    DraftingContext(
        contract_type="saas_agreement",
        party_position="pro_company",
        deal_context="Enterprise SaaS for financial analytics. Customer is a mid-size bank.",
        business_constraints=["SOC 2 Type II", "annual billing", "99.9% uptime"],
        governing_law="Delaware",
        company_name="FinAnalytics Inc",
        counterparty_name="MidSize Bank",
    ),
    DraftingContext(
        contract_type="nda",
        party_position="balanced",
        deal_context="Mutual NDA for M&A discussions between two tech companies.",
        business_constraints=["3 year term", "mutual obligations", "return of information"],
        governing_law="California",
        company_name="TechCorp A",
        counterparty_name="TechCorp B",
    ),
    DraftingContext(
        contract_type="dpa",
        party_position="balanced",
        deal_context="GDPR DPA for SaaS provider processing EU personal data.",
        business_constraints=["GDPR compliant", "subprocessor list", "audit rights"],
        governing_law="Ireland",
        company_name="CloudProvider",
        counterparty_name="EU Controller",
    ),
]

for ctx in samples:
    contract = engine.draft(ctx)
    md = engine.export(contract, fmt="markdown")
    fname = f"/app/sample_{ctx.contract_type}_{ctx.party_position}.md"
    with open(fname, "w") as f:
        f.write(md)
    print(f"  Saved {fname}")

# Generate failure report
print("\n[5] Generating failure report...")
failure_lines = ["# Failure Report", ""]
for r in results:
    failures = []
    for dim, score in r.scores.items():
        if score < 0.5:
            failures.append(f"  - {dim}: {score:.3f} (BELOW THRESHOLD)")
    if failures:
        failure_lines.append(f"## {r.task_id} ({r.contract_type})")
        failure_lines.extend(failures)
        failure_lines.append("")
        failure_lines.append("Root causes:")
        if r.scores.get("clause_completeness", 1) < 0.5:
            failure_lines.append("  - Missing required clauses - template coverage insufficient")
        if r.scores.get("business_usefulness", 1) < 0.5:
            failure_lines.append("  - Business constraints not reflected - need constraint-aware generation")
        if r.scores.get("risk_flag_accuracy", 1) < 0.5:
            failure_lines.append("  - Risk flag detection heuristics too simplistic")
        if r.scores.get("citation_support", 1) < 0.5:
            failure_lines.append("  - No retrieved precedent clauses for this clause type")
        failure_lines.append("")

if len(failure_lines) <= 2:
    failure_lines.append("No critical failures detected. All scores above 0.5 threshold.")
    failure_lines.append("")
    failure_lines.append("Known limitations:")
    failure_lines.append("  - Template-based generation produces short, generic clauses")
    failure_lines.append("  - Risk flag detection uses keyword matching, not semantic understanding")
    failure_lines.append("  - Playbook compliance scoring is heuristic-based")
    failure_lines.append("  - No LLM-based generation when running without GPU/transformers")
    failure_lines.append("  - Business constraints checked via string matching, not semantic embedding")

with open("/app/failure_report.md", "w") as f:
    f.write("\n".join(failure_lines))

print("Done! All outputs saved to /app/")