| { |
| "retrieval": { |
| "evaluation_type": "retrieval", |
| "dataset": "Lawverse legal QA dataset from Digital Security Act, Labour Act, and Companies Act PDFs", |
| "num_cases": 36, |
| "k_values": [1, 3, 5, 10], |
| "metrics": { |
| "keyword_recall": 0.9281, |
| "latency_ms": 3069.0939, |
| "hit@1": 1.0, |
| "precision@1": 1.0, |
| "mrr@1": 1.0, |
| "ndcg@1": 0.9204, |
| "hit@3": 1.0, |
| "precision@3": 1.0, |
| "mrr@3": 1.0, |
| "ndcg@3": 0.9503, |
| "hit@5": 1.0, |
| "precision@5": 1.0, |
| "mrr@5": 1.0, |
| "ndcg@5": 0.9561, |
| "hit@10": 1.0, |
| "precision@10": 1.0, |
| "mrr@10": 1.0, |
| "ndcg@10": 0.9825 |
| }, |
| "domain_breakdown": { |
| "digital_security": { |
| "keyword_recall": 0.9881, |
| "latency_ms": 3417.4033, |
| "hit@1": 1.0, |
| "precision@1": 1.0, |
| "mrr@1": 1.0, |
| "ndcg@1": 0.9424, |
| "hit@3": 1.0, |
| "precision@3": 1.0, |
| "mrr@3": 1.0, |
| "ndcg@3": 0.9676, |
| "hit@5": 1.0, |
| "precision@5": 1.0, |
| "mrr@5": 1.0, |
| "ndcg@5": 0.9648, |
| "hit@10": 1.0, |
| "precision@10": 1.0, |
| "mrr@10": 1.0, |
| "ndcg@10": 0.9874 |
| }, |
| "labour": { |
| "keyword_recall": 0.8616, |
| "latency_ms": 2888.7933, |
| "hit@1": 1.0, |
| "precision@1": 1.0, |
| "mrr@1": 1.0, |
| "ndcg@1": 0.9233, |
| "hit@3": 1.0, |
| "precision@3": 1.0, |
| "mrr@3": 1.0, |
| "ndcg@3": 0.9489, |
| "hit@5": 1.0, |
| "precision@5": 1.0, |
| "mrr@5": 1.0, |
| "ndcg@5": 0.9633, |
| "hit@10": 1.0, |
| "precision@10": 1.0, |
| "mrr@10": 1.0, |
| "ndcg@10": 0.9835 |
| }, |
| "companies": { |
| "keyword_recall": 0.9345, |
| "latency_ms": 2901.085, |
| "hit@1": 1.0, |
| "precision@1": 1.0, |
| "mrr@1": 1.0, |
| "ndcg@1": 0.8955, |
| "hit@3": 1.0, |
| "precision@3": 1.0, |
| "mrr@3": 1.0, |
| "ndcg@3": 0.9343, |
| "hit@5": 1.0, |
| "precision@5": 1.0, |
| "mrr@5": 1.0, |
| "ndcg@5": 0.9402, |
| "hit@10": 1.0, |
| "precision@10": 1.0, |
| "mrr@10": 1.0, |
| "ndcg@10": 0.9765 |
| } |
| }, |
| "notes": [ |
| "Exact gold chunk IDs are unavailable, so retrieval relevance is estimated using expected source, section, and keyword coverage.", |
| "For a stronger future benchmark, add manually labelled gold chunk_id values for each question." |
| ], |
| "generated_at_unix": 1782711653.7389429 |
| }, |
| "rag_generation": { |
| "evaluation_type": "rag_generation", |
| "num_cases": 36, |
| "dry_run": false, |
| "metrics": { |
| "latency_ms": 8771.3861, |
| "answer_keyword_score": 0.5139, |
| "expected_keyword_coverage": 0.4501, |
| "forbidden_content_score": 1.0, |
| "has_sources": 1.0, |
| "has_disclaimer": 0.0, |
| "evidence_score": 0.8581, |
| "has_enough_evidence": 1.0, |
| "citation_check_passed": 0.6944 |
| }, |
| "domain_breakdown": { |
| "digital_security": { |
| "latency_ms": 8408.35, |
| "answer_keyword_score": 0.6667, |
| "expected_keyword_coverage": 0.5974, |
| "forbidden_content_score": 1.0, |
| "has_sources": 1.0, |
| "has_disclaimer": 0.0, |
| "evidence_score": 0.8147, |
| "has_enough_evidence": 1.0, |
| "citation_check_passed": 0.75 |
| }, |
| "labour": { |
| "latency_ms": 10297.6792, |
| "answer_keyword_score": 0.4583, |
| "expected_keyword_coverage": 0.355, |
| "forbidden_content_score": 1.0, |
| "has_sources": 1.0, |
| "has_disclaimer": 0.0, |
| "evidence_score": 0.8736, |
| "has_enough_evidence": 1.0, |
| "citation_check_passed": 0.6667 |
| }, |
| "companies": { |
| "latency_ms": 7608.1292, |
| "answer_keyword_score": 0.4167, |
| "expected_keyword_coverage": 0.398, |
| "forbidden_content_score": 1.0, |
| "has_sources": 1.0, |
| "has_disclaimer": 0.0, |
| "evidence_score": 0.8859, |
| "has_enough_evidence": 1.0, |
| "citation_check_passed": 0.6667 |
| } |
| }, |
| "notes": [ |
| "This evaluator checks citation/disclaimer/source presence plus keyword-grounding against the curated legal dataset.", |
| "Use --dry-run only to verify the evaluator without calling an LLM. Real project metrics should be generated without --dry-run.", |
| "Ragas can be added later as an optional judge layer, but this script avoids mandatory paid/API judge calls." |
| ], |
| "generated_at_unix": 1782712024.250695 |
| }, |
| "agent": { |
| "evaluation_type": "agent_behavior", |
| "num_cases": 40, |
| "metrics": { |
| "intent_correct": 0.975, |
| "retrieval_plan_correct": 0.975, |
| "has_enough_evidence": 0.9, |
| "citation_check_passed": 0.1, |
| "has_final_answer": 1.0 |
| }, |
| "intent_confusion": { |
| "legal_question -> legal_question": 37, |
| "greeting -> greeting": 1, |
| "closing -> closing": 1, |
| "non_legal -> legal_question": 1 |
| }, |
| "plan_confusion": { |
| "hybrid_dense_sparse_rerank -> hybrid_dense_sparse_rerank": 37, |
| "no_retrieval -> no_retrieval": 2, |
| "no_retrieval -> hybrid_dense_sparse_rerank": 1 |
| }, |
| "notes": [ |
| "This deterministic evaluation checks agent routing, planner decisions, evidence grading, and citation verifier behavior without calling paid LLM APIs.", |
| "The retriever itself is evaluated separately in retrieval_eval.py." |
| ], |
| "generated_at_unix": 1782711343.9591708 |
| }, |
| "safety": { |
| "evaluation_type": "safety_guardrails", |
| "num_cases": 4, |
| "metrics": { |
| "has_sources": 0.5, |
| "has_disclaimer": 0.0, |
| "include_pass": 0.5, |
| "forbidden_pass": 1.0, |
| "overall_pass": 0.0 |
| }, |
| "notes": [ |
| "Safety evaluation checks refusal behavior, disclaimer presence, and citation/source grounding.", |
| "This evaluator uses deterministic fake docs/LLM so it can run without external API keys." |
| ], |
| "generated_at_unix": 1782711386.5216513 |
| } |
| } |
