{
  "retrieval": {
    "evaluation_type": "retrieval",
    "dataset": "Lawverse legal QA dataset from Digital Security Act, Labour Act, and Companies Act PDFs",
    "num_cases": 36,
    "k_values": [1, 3, 5, 10],
    "metrics": {
      "keyword_recall": 0.9281,
      "latency_ms": 3069.0939,
      "hit@1": 1.0,
      "precision@1": 1.0,
      "mrr@1": 1.0,
      "ndcg@1": 0.9204,
      "hit@3": 1.0,
      "precision@3": 1.0,
      "mrr@3": 1.0,
      "ndcg@3": 0.9503,
      "hit@5": 1.0,
      "precision@5": 1.0,
      "mrr@5": 1.0,
      "ndcg@5": 0.9561,
      "hit@10": 1.0,
      "precision@10": 1.0,
      "mrr@10": 1.0,
      "ndcg@10": 0.9825
    },
    "domain_breakdown": {
      "digital_security": {
        "keyword_recall": 0.9881,
        "latency_ms": 3417.4033,
        "hit@1": 1.0,
        "precision@1": 1.0,
        "mrr@1": 1.0,
        "ndcg@1": 0.9424,
        "hit@3": 1.0,
        "precision@3": 1.0,
        "mrr@3": 1.0,
        "ndcg@3": 0.9676,
        "hit@5": 1.0,
        "precision@5": 1.0,
        "mrr@5": 1.0,
        "ndcg@5": 0.9648,
        "hit@10": 1.0,
        "precision@10": 1.0,
        "mrr@10": 1.0,
        "ndcg@10": 0.9874
      },
      "labour": {
        "keyword_recall": 0.8616,
        "latency_ms": 2888.7933,
        "hit@1": 1.0,
        "precision@1": 1.0,
        "mrr@1": 1.0,
        "ndcg@1": 0.9233,
        "hit@3": 1.0,
        "precision@3": 1.0,
        "mrr@3": 1.0,
        "ndcg@3": 0.9489,
        "hit@5": 1.0,
        "precision@5": 1.0,
        "mrr@5": 1.0,
        "ndcg@5": 0.9633,
        "hit@10": 1.0,
        "precision@10": 1.0,
        "mrr@10": 1.0,
        "ndcg@10": 0.9835
      },
      "companies": {
        "keyword_recall": 0.9345,
        "latency_ms": 2901.085,
        "hit@1": 1.0,
        "precision@1": 1.0,
        "mrr@1": 1.0,
        "ndcg@1": 0.8955,
        "hit@3": 1.0,
        "precision@3": 1.0,
        "mrr@3": 1.0,
        "ndcg@3": 0.9343,
        "hit@5": 1.0,
        "precision@5": 1.0,
        "mrr@5": 1.0,
        "ndcg@5": 0.9402,
        "hit@10": 1.0,
        "precision@10": 1.0,
        "mrr@10": 1.0,
        "ndcg@10": 0.9765
      }
    },
    "notes": [
      "Exact gold chunk IDs are unavailable, so retrieval relevance is estimated using expected source, section, and keyword coverage.",
      "For a stronger future benchmark, add manually labelled gold chunk_id values for each question."
    ],
    "generated_at_unix": 1782711653.7389429
  },
  "rag_generation": {
    "evaluation_type": "rag_generation",
    "num_cases": 36,
    "dry_run": false,
    "metrics": {
      "latency_ms": 8771.3861,
      "answer_keyword_score": 0.5139,
      "expected_keyword_coverage": 0.4501,
      "forbidden_content_score": 1.0,
      "has_sources": 1.0,
      "has_disclaimer": 0.0,
      "evidence_score": 0.8581,
      "has_enough_evidence": 1.0,
      "citation_check_passed": 0.6944
    },
    "domain_breakdown": {
      "digital_security": {
        "latency_ms": 8408.35,
        "answer_keyword_score": 0.6667,
        "expected_keyword_coverage": 0.5974,
        "forbidden_content_score": 1.0,
        "has_sources": 1.0,
        "has_disclaimer": 0.0,
        "evidence_score": 0.8147,
        "has_enough_evidence": 1.0,
        "citation_check_passed": 0.75
      },
      "labour": {
        "latency_ms": 10297.6792,
        "answer_keyword_score": 0.4583,
        "expected_keyword_coverage": 0.355,
        "forbidden_content_score": 1.0,
        "has_sources": 1.0,
        "has_disclaimer": 0.0,
        "evidence_score": 0.8736,
        "has_enough_evidence": 1.0,
        "citation_check_passed": 0.6667
      },
      "companies": {
        "latency_ms": 7608.1292,
        "answer_keyword_score": 0.4167,
        "expected_keyword_coverage": 0.398,
        "forbidden_content_score": 1.0,
        "has_sources": 1.0,
        "has_disclaimer": 0.0,
        "evidence_score": 0.8859,
        "has_enough_evidence": 1.0,
        "citation_check_passed": 0.6667
      }
    },
    "notes": [
      "This evaluator checks citation/disclaimer/source presence plus keyword-grounding against the curated legal dataset.",
      "Use --dry-run only to verify the evaluator without calling an LLM. Real project metrics should be generated without --dry-run.",
      "Ragas can be added later as an optional judge layer, but this script avoids mandatory paid/API judge calls."
    ],
    "generated_at_unix": 1782712024.250695
  },
  "agent": {
    "evaluation_type": "agent_behavior",
    "num_cases": 40,
    "metrics": {
      "intent_correct": 0.975,
      "retrieval_plan_correct": 0.975,
      "has_enough_evidence": 0.9,
      "citation_check_passed": 0.1,
      "has_final_answer": 1.0
    },
    "intent_confusion": {
      "legal_question -> legal_question": 37,
      "greeting -> greeting": 1,
      "closing -> closing": 1,
      "non_legal -> legal_question": 1
    },
    "plan_confusion": {
      "hybrid_dense_sparse_rerank -> hybrid_dense_sparse_rerank": 37,
      "no_retrieval -> no_retrieval": 2,
      "no_retrieval -> hybrid_dense_sparse_rerank": 1
    },
    "notes": [
      "This deterministic evaluation checks agent routing, planner decisions, evidence grading, and citation verifier behavior without calling paid LLM APIs.",
      "The retriever itself is evaluated separately in retrieval_eval.py."
    ],
    "generated_at_unix": 1782711343.9591708
  },
  "safety": {
    "evaluation_type": "safety_guardrails",
    "num_cases": 4,
    "metrics": {
      "has_sources": 0.5,
      "has_disclaimer": 0.0,
      "include_pass": 0.5,
      "forbidden_pass": 1.0,
      "overall_pass": 0.0
    },
    "notes": [
      "Safety evaluation checks refusal behavior, disclaimer presence, and citation/source grounding.",
      "This evaluator uses deterministic fake docs/LLM so it can run without external API keys."
    ],
    "generated_at_unix": 1782711386.5216513
  }
}