{
  "summary": {
    "ran_at": "2026-05-14T10:56:52Z",
    "elapsed_seconds": 62.3,
    "n_questions": 5,
    "factual_accuracy": 0.6,
    "citation_accuracy": 1.0,
    "refusal_precision": 0.0,
    "by_type": {
      "waiting_period": 1.0,
      "coverage_scope": 0.0,
      "sub_limit": 1.0,
      "exclusions_oos": 0.0
    },
    "by_brain": {
      "nim-chain": 0.6
    },
    "blocked_count": 0
  },
  "results": [
    {
      "id": "aditya-birla__activ-assure-diamond__wordings::pre_existing_disease_waiting_months::easy",
      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
      "question": "What is the waiting period for pre-existing diseases under Activ Assure?",
      "expected_answer": "24 months from policy inception",
      "bot_answer": "Under Activ Assure Diamond, the waiting period for pre-existing diseases is 24 months if continuous coverage has been maintained since the first policy with Aditya Birla Health Insurance [Source: Activ Assure Diamond (aditya-birla), pp.10-11]. This applies only if the optional cover was selected at the inception of the first policy and no break in coverage occurred. Waiting period credits from prior policies may be transferred under IRDAI portability rules [Source: Activ Assure Diamond (aditya-birla), p.17]. Note: This waiting period applies only to the base sum insured \u2014 any increase at renewal may be subject to new waiting periods. I'd recommend you confirm with the insurer directly before finalizing.",
      "factual_match": true,
      "citation_present": true,
      "judge_score": 1.0,
      "judge_reason": "regex: matched_nums=['24']",
      "expected_refusal": false,
      "question_type": "waiting_period",
      "difficulty": "easy",
      "blocked": false,
      "faithfulness_passed": true,
      "faithfulness_reasons": [],
      "brain_used": "nim-chain::qwen3-next-80b-a3b-instruct::v4-flash::qa",
      "latency_ms": 20509
    },
    {
      "id": "aditya-birla__activ-assure-diamond__wordings::pre_existing_disease_waiting_months::medium",
      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
      "question": "If I have diabetes, how long do I have to wait before I can claim under Activ Assure?",
      "expected_answer": "24 months \u2014 pre-existing diseases have a waiting period of 24 months from policy start",
      "bot_answer": "If you have diabetes, the waiting period for claiming under Activ Assure Diamond is 24 months, provided you\u2019ve had continuous coverage without a break since your first policy with Aditya Birla Health Insurance [Source: Activ Assure Diamond (aditya-birla), p.11]. This applies to pre-existing diseases like diabetes, and the waiting period is reduced from the standard 48 months to 24 months under portability or continuous renewal terms [Source: Activ Assure Diamond (aditya-birla), p.11]. Note: This benefit applies only if you\u2019ve maintained uninterrupted coverage with the insurer. I'd recommend you confirm with the insurer directly before finalizing.",
      "factual_match": true,
      "citation_present": true,
      "judge_score": 1.0,
      "judge_reason": "regex: matched_nums=['24']",
      "expected_refusal": false,
      "question_type": "waiting_period",
      "difficulty": "medium",
      "blocked": false,
      "faithfulness_passed": true,
      "faithfulness_reasons": [],
      "brain_used": "nim-chain::qwen3-next-80b-a3b-instruct::v4-flash::qa",
      "latency_ms": 7926
    },
    {
      "id": "aditya-birla__activ-assure-diamond__wordings::ayush_coverage::easy",
      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
      "question": "Does Activ Assure cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
      "expected_answer": "Yes",
      "bot_answer": "Yes, Activ Assure covers AYUSH treatments under in-patient hospitalization, including Ayurveda, Yoga, Unani, Siddha, and Homeopathy, on a reimbursement basis, subject to limits in the Policy Schedule [Source: Activ Assure Diamond (aditya-birla), p.4]. The treatment must be received at approved facilities such as government hospitals, AYUSH colleges recognized by CCIM/CCH, or institutes accredited by QCI/NABH, and administered by a licensed AYUSH practitioner [Source: Activ Assure Diamond (aditya-birla), p.4]. Note: Pre- and post-hospitalization expenses for AYUSH are not covered [Source: Activ Assure Diamond (aditya-birla), p.4]. I'd recommend you confirm with the insurer directly before finalizing.",
      "factual_match": false,
      "citation_present": true,
      "judge_score": 0.0,
      "judge_reason": "regex: no_overlap (gold_words=[])",
      "expected_refusal": false,
      "question_type": "coverage_scope",
      "difficulty": "easy",
      "blocked": false,
      "faithfulness_passed": true,
      "faithfulness_reasons": [],
      "brain_used": "nim-chain::qwen3-next-80b-a3b-instruct::v4-flash::qa",
      "latency_ms": 8595
    },
    {
      "id": "aditya-birla__activ-assure-diamond__wordings::room_rent_capping::medium",
      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
      "question": "Is there a cap on room rent under Activ Assure?",
      "expected_answer": "Single Private A/C Room (upgradable to next level, only if Single Private A/C Room is not available)",
      "bot_answer": "Yes, there is a cap on room rent under Activ Assure \u2014 it is limited to the \"Reasonable and Customary Charges\" for a Single Private A/C Room, as specified in the Policy Schedule or Product Benefit Table [Source: Activ Assure Diamond (aditya-birla), p.1]. The exact limit depends on your chosen plan and is not a fixed percentage of the sum insured \u2014 you must check your Policy Schedule for the applicable room rent sub-limit [Source: Activ Assure Diamond (aditya-birla), p.1]. Note: This cap applies to room rent, and other expenses like ICU or surgeon fees are covered separately, subject to their own limits. I'd recommend you confirm with the insurer directly before finalizing.",
      "factual_match": true,
      "citation_present": true,
      "judge_score": 1.0,
      "judge_reason": "regex: keyword_overlap=['private', 'room', 'single']",
      "expected_refusal": false,
      "question_type": "sub_limit",
      "difficulty": "medium",
      "blocked": false,
      "faithfulness_passed": true,
      "faithfulness_reasons": [],
      "brain_used": "nim-chain::qwen3-next-80b-a3b-instruct::v4-flash::qa",
      "latency_ms": 12799
    },
    {
      "id": "aditya-birla__activ-assure-diamond__wordings::REFUSE::exclusions_oos::hard",
      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
      "question": "Does Activ Assure cover injuries from space tourism?",
      "expected_answer": "Bot should refuse or say not in document.",
      "bot_answer": "Activ Assure does not cover injuries from space tourism. The policy excludes participation in aviation, parachuting, skydiving, and other high-risk activities [Source: Activ Assure Diamond (aditya-birla), p.13], and space tourism would fall under extreme, non-commercial aviation risks not contemplated in any covered benefit. Additionally, no provision exists for extraterrestrial travel or related injuries. I'd recommend you confirm with the insurer directly before finalizing.",
      "factual_match": false,
      "citation_present": true,
      "judge_score": 0.0,
      "judge_reason": "did not refuse when expected",
      "expected_refusal": true,
      "question_type": "exclusions_oos",
      "difficulty": "hard",
      "blocked": false,
      "faithfulness_passed": true,
      "faithfulness_reasons": [],
      "brain_used": "nim-chain::qwen3-next-80b-a3b-instruct::v4-flash::qa",
      "latency_ms": 12511
    }
  ]
}