Spaces:

rohitsar567
/

InsuranceBot

Sleeping

App Files Files Community

rohitsar567 commited on May 12

Commit

3e6b7ee

verified ·

1 Parent(s): 5dbdf6e

Deploy v1 — single-Docker FastAPI + Next.js + RAG + voice + faithfulness

Browse files

Files changed (8) hide show

Dockerfile +6 -2
backend/faithfulness.py +4 -2
entrypoint.sh +23 -10
eval/gold_qa.json +1124 -0
eval/results.json +525 -0
eval/results.md +55 -0
rag/vectors/chroma.sqlite3 +1 -1
tools/set_hf_secrets.py +58 -0

Dockerfile CHANGED Viewed

@@ -54,9 +54,13 @@ COPY --from=frontend-builder /app/frontend/out ./frontend/out
 ENV PORT=7860
 EXPOSE 7860
 # Use a non-root user (HF Spaces recommends this for Docker spaces)
 RUN useradd -m -u 1000 user && chown -R user:user /app
 USER user
-# Start the FastAPI app
-CMD ["sh", "-c", "uvicorn backend.main:app --host 0.0.0.0 --port ${PORT} --log-level info"]

 ENV PORT=7860
 EXPOSE 7860
+# Copy entrypoint and make it executable
+COPY entrypoint.sh ./entrypoint.sh
+RUN chmod +x ./entrypoint.sh
 # Use a non-root user (HF Spaces recommends this for Docker spaces)
 RUN useradd -m -u 1000 user && chown -R user:user /app
 USER user
+# Start: entrypoint validates Chroma + (re-)ingests if needed, then runs uvicorn
+CMD ["sh", "/app/entrypoint.sh"]

backend/faithfulness.py CHANGED Viewed

@@ -53,8 +53,10 @@ HALLUCINATION_LOG = LOG_DIR / "hallucinations.jsonl"
 # Tunables — these become evaluable parameters in the eval harness.
 # BGE-small returns higher cosine similarity than Voyage, so the floors are
 # higher here than they would be for Voyage. Re-tune if changing embedding model.
-MIN_TOP_SCORE = 0.40  # below this we refuse outright (BGE-small cosine similarity)
-MIN_AVG_SCORE = 0.30  # average of top 5 must be above this
 @dataclass

 # Tunables — these become evaluable parameters in the eval harness.
 # BGE-small returns higher cosine similarity than Voyage, so the floors are
 # higher here than they would be for Voyage. Re-tune if changing embedding model.
+# Lowered 2026-05-13 based on eval data showing too-aggressive refusal at 0.40:
+# many real questions retrieve top chunks at 0.30-0.38 that DO contain the answer.
+MIN_TOP_SCORE = 0.30  # below this we refuse outright (BGE-small cosine similarity)
+MIN_AVG_SCORE = 0.22  # average of top 5 must be above this
 @dataclass

entrypoint.sh CHANGED Viewed

@@ -23,19 +23,32 @@ if [ -d "/data" ] && [ -w "/data" ]; then
     ln -sf /data/policies.duckdb /app/rag/policies.duckdb
 fi
-# Ingest if Chroma collection is empty
-echo "[entrypoint] checking if Chroma needs to be built..."
 python -c "
 import sys
 sys.path.insert(0, '/app')
-from rag.retrieve import get_collection
-c = get_collection()
-n = c.count()
-print(f'[entrypoint] Chroma chunks: {n}')
-sys.exit(0 if n > 0 else 1)
-" && echo "[entrypoint] Chroma already populated — skipping ingest" || (
-    echo "[entrypoint] Chroma empty — running ingest (~5 min one-time)..."
-    python -m rag.ingest 2>&1 | tail -20
 )
 # Start the server

     ln -sf /data/policies.duckdb /app/rag/policies.duckdb
 fi
+# Validate Chroma is readable + populated; rebuild if not.
+echo "[entrypoint] validating Chroma vector store..."
 python -c "
 import sys
 sys.path.insert(0, '/app')
+try:
+    from rag.retrieve import get_collection
+    c = get_collection()
+    n = c.count()
+    if n <= 0:
+        print(f'[entrypoint] Chroma is empty')
+        sys.exit(1)
+    # Smoke test: do an actual retrieval to surface any deserialization bug
+    res = c.get(limit=1, include=['metadatas'])
+    if not res.get('ids'):
+        print(f'[entrypoint] Chroma reports {n} chunks but get() returns empty')
+        sys.exit(1)
+    print(f'[entrypoint] Chroma OK: {n} chunks, sample policy: {res[\"metadatas\"][0].get(\"policy_id\")}')
+    sys.exit(0)
+except Exception as e:
+    print(f'[entrypoint] Chroma load FAILED: {type(e).__name__}: {e}')
+    sys.exit(1)
+" || (
+    echo "[entrypoint] Wiping vector store and re-ingesting from corpus..."
+    rm -rf /app/rag/vectors/* 2>/dev/null || true
+    python -m rag.ingest 2>&1 | tail -30
 )
 # Start the server

eval/gold_qa.json ADDED Viewed

	@@ -0,0 +1,1124 @@

+[
+  {
+    "id": "aditya-birla__activ-assure-diamond__wordings::pre_existing_disease_waiting_months::easy",
+    "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+    "question": "What is the waiting period for pre-existing diseases under Activ Assure?",
+    "expected_answer": "24 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "aditya-birla__activ-assure-diamond__wordings::pre_existing_disease_waiting_months::medium",
+    "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Activ Assure?",
+    "expected_answer": "24 months \u2014 pre-existing diseases have a waiting period of 24 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "aditya-birla__activ-assure-diamond__wordings::ayush_coverage::easy",
+    "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+    "question": "Does Activ Assure cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+    "expected_answer": "Yes",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "ayush_coverage"
+  },
+  {
+    "id": "aditya-birla__activ-assure-diamond__wordings::room_rent_capping::medium",
+    "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+    "question": "Is there a cap on room rent under Activ Assure?",
+    "expected_answer": "Single Private A/C Room (upgradable to next level, only if Single Private A/C Room is not available)",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "aditya-birla__activ-assure-diamond__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+    "question": "Does Activ Assure cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "aditya-birla__activ-assure-diamond__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Activ Assure?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "aditya-birla__activ-assure-diamond__wordings::REFUSE::regulatory_oos::hard",
+    "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+    "question": "What is the IRDAI mandate on dental coverage that Activ Assure must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__comprehensive-care-plan__wordings::pre_existing_disease_waiting_months::easy",
+    "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+    "question": "What is the waiting period for pre-existing diseases under Comprehensive Care Plan?",
+    "expected_answer": "36 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "bajaj-allianz__comprehensive-care-plan__wordings::pre_existing_disease_waiting_months::medium",
+    "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Comprehensive Care Plan?",
+    "expected_answer": "36 months \u2014 pre-existing diseases have a waiting period of 36 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "bajaj-allianz__comprehensive-care-plan__wordings::initial_waiting_period_days::easy",
+    "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+    "question": "What is the initial waiting period under Comprehensive Care Plan?",
+    "expected_answer": "90 days from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "initial_waiting_period_days"
+  },
+  {
+    "id": "bajaj-allianz__comprehensive-care-plan__wordings::ayush_coverage::easy",
+    "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+    "question": "Does Comprehensive Care Plan cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+    "expected_answer": "Yes",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "ayush_coverage"
+  },
+  {
+    "id": "bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+    "question": "Does Comprehensive Care Plan cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Comprehensive Care Plan?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::regulatory_oos::hard",
+    "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+    "question": "What is the IRDAI mandate on dental coverage that Comprehensive Care Plan must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::pre_existing_disease_waiting_months::easy",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "What is the waiting period for pre-existing diseases under Silver Health?",
+    "expected_answer": "24 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::pre_existing_disease_waiting_months::medium",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Silver Health?",
+    "expected_answer": "24 months \u2014 pre-existing diseases have a waiting period of 24 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::initial_waiting_period_days::easy",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "What is the initial waiting period under Silver Health?",
+    "expected_answer": "30 days from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "initial_waiting_period_days"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::pre_hospitalization_days::easy",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "How many days of pre-hospitalization expenses does Silver Health cover?",
+    "expected_answer": "30 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_hospitalization_days"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::post_hospitalization_days::easy",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "How many days of post-hospitalization expenses does Silver Health cover?",
+    "expected_answer": "60 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "post_hospitalization_days"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::no_claim_bonus_pct::easy",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "What's the no-claim bonus on Silver Health?",
+    "expected_answer": "10.0% step-up on sum insured per claim-free year",
+    "question_type": "bonus",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "no_claim_bonus_pct"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::room_rent_capping::medium",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "Is there a cap on room rent under Silver Health?",
+    "expected_answer": "1% of hospitalization Sum Insured up to maximum Rs. 7,500 per day",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::copayment_pct::easy",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "Is there a copayment under Silver Health?",
+    "expected_answer": "10% copayment applies",
+    "question_type": "sub_limit",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "copayment_pct"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::REFUSE::exclusions_oos::hard",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "Does Silver Health cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::REFUSE::exclusions_oos::hard",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Silver Health?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__silver-health__cis::REFUSE::regulatory_oos::hard",
+    "policy_id": "bajaj-allianz__silver-health__cis",
+    "question": "What is the IRDAI mandate on dental coverage that Silver Health must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__tax-gain__cis::pre_existing_disease_waiting_months::easy",
+    "policy_id": "bajaj-allianz__tax-gain__cis",
+    "question": "What is the waiting period for pre-existing diseases under Tax Gain?",
+    "expected_answer": "36 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "bajaj-allianz__tax-gain__cis::pre_existing_disease_waiting_months::medium",
+    "policy_id": "bajaj-allianz__tax-gain__cis",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Tax Gain?",
+    "expected_answer": "36 months \u2014 pre-existing diseases have a waiting period of 36 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "bajaj-allianz__tax-gain__cis::initial_waiting_period_days::easy",
+    "policy_id": "bajaj-allianz__tax-gain__cis",
+    "question": "What is the initial waiting period under Tax Gain?",
+    "expected_answer": "30 days from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "initial_waiting_period_days"
+  },
+  {
+    "id": "bajaj-allianz__tax-gain__cis::REFUSE::exclusions_oos::hard",
+    "policy_id": "bajaj-allianz__tax-gain__cis",
+    "question": "Does Tax Gain cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__tax-gain__cis::REFUSE::exclusions_oos::hard",
+    "policy_id": "bajaj-allianz__tax-gain__cis",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Tax Gain?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "bajaj-allianz__tax-gain__cis::REFUSE::regulatory_oos::hard",
+    "policy_id": "bajaj-allianz__tax-gain__cis",
+    "question": "What is the IRDAI mandate on dental coverage that Tax Gain must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::pre_existing_disease_waiting_months::easy",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "What is the waiting period for pre-existing diseases under Care Advantage?",
+    "expected_answer": "36 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::pre_existing_disease_waiting_months::medium",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Care Advantage?",
+    "expected_answer": "36 months \u2014 pre-existing diseases have a waiting period of 36 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::initial_waiting_period_days::easy",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "What is the initial waiting period under Care Advantage?",
+    "expected_answer": "30 days from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "initial_waiting_period_days"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::pre_hospitalization_days::easy",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "How many days of pre-hospitalization expenses does Care Advantage cover?",
+    "expected_answer": "30 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::post_hospitalization_days::easy",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "How many days of post-hospitalization expenses does Care Advantage cover?",
+    "expected_answer": "60 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "post_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::ayush_coverage::easy",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "Does Care Advantage cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+    "expected_answer": "Yes",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "ayush_coverage"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::no_claim_bonus_pct::easy",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "What's the no-claim bonus on Care Advantage?",
+    "expected_answer": "10.0% step-up on sum insured per claim-free year",
+    "question_type": "bonus",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "no_claim_bonus_pct"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::room_rent_capping::medium",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "Is there a cap on room rent under Care Advantage?",
+    "expected_answer": "No sub-limit",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::copayment_pct::easy",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "Is there a copayment under Care Advantage?",
+    "expected_answer": "20% copayment applies",
+    "question_type": "sub_limit",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "copayment_pct"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "Does Care Advantage cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Care Advantage?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure::REFUSE::regulatory_oos::hard",
+    "policy_id": "care-health__care-advantage-add-ons-protect-plus-care-shield__brochure",
+    "question": "What is the IRDAI mandate on dental coverage that Care Advantage must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-classic__wordings::pre_existing_disease_waiting_months::easy",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "What is the waiting period for pre-existing diseases under Care Classic?",
+    "expected_answer": "36 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-classic__wordings::pre_existing_disease_waiting_months::medium",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Care Classic?",
+    "expected_answer": "36 months \u2014 pre-existing diseases have a waiting period of 36 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-classic__wordings::maternity_waiting_months::easy",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "What's the maternity benefit waiting period under Care Classic?",
+    "expected_answer": "36 months",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "maternity_waiting_months"
+  },
+  {
+    "id": "care-health__care-classic__wordings::pre_hospitalization_days::easy",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "How many days of pre-hospitalization expenses does Care Classic cover?",
+    "expected_answer": "60 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-classic__wordings::post_hospitalization_days::easy",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "How many days of post-hospitalization expenses does Care Classic cover?",
+    "expected_answer": "90 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "post_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-classic__wordings::ayush_coverage::easy",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "Does Care Classic cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+    "expected_answer": "Yes",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "ayush_coverage"
+  },
+  {
+    "id": "care-health__care-classic__wordings::no_claim_bonus_pct::easy",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "What's the no-claim bonus on Care Classic?",
+    "expected_answer": "25.0% step-up on sum insured per claim-free year",
+    "question_type": "bonus",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "no_claim_bonus_pct"
+  },
+  {
+    "id": "care-health__care-classic__wordings::room_rent_capping::medium",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "Is there a cap on room rent under Care Classic?",
+    "expected_answer": "1% of SI per day or Single Private AC Room",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "care-health__care-classic__wordings::copayment_pct::easy",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "Is there a copayment under Care Classic?",
+    "expected_answer": "20% copayment applies",
+    "question_type": "sub_limit",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "copayment_pct"
+  },
+  {
+    "id": "care-health__care-classic__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "Does Care Classic cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-classic__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Care Classic?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-classic__wordings::REFUSE::regulatory_oos::hard",
+    "policy_id": "care-health__care-classic__wordings",
+    "question": "What is the IRDAI mandate on dental coverage that Care Classic must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-senior__brochure::pre_existing_disease_waiting_months::easy",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "What is the waiting period for pre-existing diseases under Care for Senior Citizens?",
+    "expected_answer": "48 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-senior__brochure::pre_existing_disease_waiting_months::medium",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Care for Senior Citizens?",
+    "expected_answer": "48 months \u2014 pre-existing diseases have a waiting period of 48 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-senior__brochure::initial_waiting_period_days::easy",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "What is the initial waiting period under Care for Senior Citizens?",
+    "expected_answer": "30 days from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "initial_waiting_period_days"
+  },
+  {
+    "id": "care-health__care-senior__brochure::pre_hospitalization_days::easy",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "How many days of pre-hospitalization expenses does Care for Senior Citizens cover?",
+    "expected_answer": "30 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-senior__brochure::post_hospitalization_days::easy",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "How many days of post-hospitalization expenses does Care for Senior Citizens cover?",
+    "expected_answer": "60 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "post_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-senior__brochure::no_claim_bonus_pct::easy",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "What's the no-claim bonus on Care for Senior Citizens?",
+    "expected_answer": "10.0% step-up on sum insured per claim-free year",
+    "question_type": "bonus",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "no_claim_bonus_pct"
+  },
+  {
+    "id": "care-health__care-senior__brochure::room_rent_capping::medium",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "Is there a cap on room rent under Care for Senior Citizens?",
+    "expected_answer": "1% SI per day (Max. up to 1% of SI per day) for 3 Lacs plan, Single Private AC Room (Max. up to 1% of SI per day) for 5,7,10 Lacs plan",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "care-health__care-senior__brochure::copayment_pct::easy",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "Is there a copayment under Care for Senior Citizens?",
+    "expected_answer": "20% copayment applies",
+    "question_type": "sub_limit",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "copayment_pct"
+  },
+  {
+    "id": "care-health__care-senior__brochure::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "Does Care for Senior Citizens cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-senior__brochure::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Care for Senior Citizens?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-senior__brochure::REFUSE::regulatory_oos::hard",
+    "policy_id": "care-health__care-senior__brochure",
+    "question": "What is the IRDAI mandate on dental coverage that Care for Senior Citizens must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::pre_existing_disease_waiting_months::easy",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "What is the waiting period for pre-existing diseases under Care Supreme?",
+    "expected_answer": "36 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::pre_existing_disease_waiting_months::medium",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Care Supreme?",
+    "expected_answer": "36 months \u2014 pre-existing diseases have a waiting period of 36 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::pre_hospitalization_days::easy",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "How many days of pre-hospitalization expenses does Care Supreme cover?",
+    "expected_answer": "60 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::post_hospitalization_days::easy",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "How many days of post-hospitalization expenses does Care Supreme cover?",
+    "expected_answer": "180 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "post_hospitalization_days"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::ayush_coverage::easy",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "Does Care Supreme cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+    "expected_answer": "Yes",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "ayush_coverage"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::no_claim_bonus_pct::easy",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "What's the no-claim bonus on Care Supreme?",
+    "expected_answer": "50.0% step-up on sum insured per claim-free year",
+    "question_type": "bonus",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "no_claim_bonus_pct"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::room_rent_capping::medium",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "Is there a cap on room rent under Care Supreme?",
+    "expected_answer": "No limit",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "Does Care Supreme cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Care Supreme?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "care-health__care-supreme__wordings::REFUSE::regulatory_oos::hard",
+    "policy_id": "care-health__care-supreme__wordings",
+    "question": "What is the IRDAI mandate on dental coverage that Care Supreme must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::pre_existing_disease_waiting_months::easy",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "What is the waiting period for pre-existing diseases under Elevate?",
+    "expected_answer": "36 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::pre_existing_disease_waiting_months::medium",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Elevate?",
+    "expected_answer": "36 months \u2014 pre-existing diseases have a waiting period of 36 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::initial_waiting_period_days::easy",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "What is the initial waiting period under Elevate?",
+    "expected_answer": "30 days from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "initial_waiting_period_days"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::pre_hospitalization_days::easy",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "How many days of pre-hospitalization expenses does Elevate cover?",
+    "expected_answer": "90 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_hospitalization_days"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::post_hospitalization_days::easy",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "How many days of post-hospitalization expenses does Elevate cover?",
+    "expected_answer": "180 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "post_hospitalization_days"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::ayush_coverage::easy",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "Does Elevate cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+    "expected_answer": "Yes",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "ayush_coverage"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::no_claim_bonus_pct::easy",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "What's the no-claim bonus on Elevate?",
+    "expected_answer": "20.0% step-up on sum insured per claim-free year",
+    "question_type": "bonus",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "no_claim_bonus_pct"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::room_rent_capping::medium",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "Is there a cap on room rent under Elevate?",
+    "expected_answer": "Single Private AC room",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "Does Elevate cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Elevate?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "icici-lombard__elevate__wordings::REFUSE::regulatory_oos::hard",
+    "policy_id": "icici-lombard__elevate__wordings",
+    "question": "What is the IRDAI mandate on dental coverage that Elevate must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::pre_existing_disease_waiting_months::easy",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "What is the waiting period for pre-existing diseases under Health AdvantEdge?",
+    "expected_answer": "48 months from policy inception",
+    "question_type": "waiting_period",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::pre_existing_disease_waiting_months::medium",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "If I have diabetes, how long do I have to wait before I can claim under Health AdvantEdge?",
+    "expected_answer": "48 months \u2014 pre-existing diseases have a waiting period of 48 months from policy start",
+    "question_type": "waiting_period",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_existing_disease_waiting_months"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::pre_hospitalization_days::easy",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "How many days of pre-hospitalization expenses does Health AdvantEdge cover?",
+    "expected_answer": "60 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "pre_hospitalization_days"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::post_hospitalization_days::easy",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "How many days of post-hospitalization expenses does Health AdvantEdge cover?",
+    "expected_answer": "180 days",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "post_hospitalization_days"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::ayush_coverage::easy",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "Does Health AdvantEdge cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+    "expected_answer": "Yes",
+    "question_type": "coverage_scope",
+    "difficulty": "easy",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "ayush_coverage"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::room_rent_capping::medium",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "Is there a cap on room rent under Health AdvantEdge?",
+    "expected_answer": "1% of Annual Sum Insured for policies up to Rs. 4 Lakhs, no capping for Rs. 5 Lakhs and above",
+    "question_type": "sub_limit",
+    "difficulty": "medium",
+    "expected_refusal": false,
+    "language": "en",
+    "generated_by": "pipeline_a",
+    "source_field": "room_rent_capping"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "Does Health AdvantEdge cover injuries from space tourism?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::REFUSE::exclusions_oos::hard",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Health AdvantEdge?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "exclusions_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  },
+  {
+    "id": "icici-lombard__health-advantedge__wordings::REFUSE::regulatory_oos::hard",
+    "policy_id": "icici-lombard__health-advantedge__wordings",
+    "question": "What is the IRDAI mandate on dental coverage that Health AdvantEdge must follow?",
+    "expected_answer": "Bot should refuse or say not in document.",
+    "question_type": "regulatory_oos",
+    "difficulty": "hard",
+    "expected_refusal": true,
+    "language": "en",
+    "generated_by": "pipeline_c_refusal"
+  }
+]

eval/results.json ADDED Viewed

	@@ -0,0 +1,525 @@

+{
+  "summary": {
+    "ran_at": "2026-05-12T22:30:15Z",
+    "elapsed_seconds": 293.0,
+    "n_questions": 25,
+    "factual_accuracy": 0.4,
+    "citation_accuracy": 0.5,
+    "refusal_precision": 0.444,
+    "by_type": {
+      "waiting_period": 0.125,
+      "coverage_scope": 1.0,
+      "sub_limit": 0.333,
+      "exclusions_oos": 0.333,
+      "regulatory_oos": 0.667,
+      "bonus": 0.0
+    },
+    "by_brain": {
+      "groq-llama": 1.0,
+      "sarvam-m": 0.375
+    },
+    "blocked_count": 12
+  },
+  "results": [
+    {
+      "id": "aditya-birla__activ-assure-diamond__wordings::pre_existing_disease_waiting_months::easy",
+      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+      "question": "What is the waiting period for pre-existing diseases under Activ Assure?",
+      "expected_answer": "24 months from policy inception",
+      "bot_answer": "The waiting period for pre-existing diseases under Activ Assure is 24 months [Source: Activ Assure Diamond (aditya-birla), pp.10-11].",
+      "factual_match": true,
+      "citation_present": true,
+      "judge_score": 1.0,
+      "judge_reason": "BOT accurately conveys the waiting period.",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "easy",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "groq-llama::fallback-truncated-reasoning",
+      "latency_ms": 31175
+    },
+    {
+      "id": "aditya-birla__activ-assure-diamond__wordings::pre_existing_disease_waiting_months::medium",
+      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+      "question": "If I have diabetes, how long do I have to wait before I can claim under Activ Assure?",
+      "expected_answer": "24 months \u2014 pre-existing diseases have a waiting period of 24 months from policy start",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "medium",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10979
+    },
+    {
+      "id": "aditya-birla__activ-assure-diamond__wordings::ayush_coverage::easy",
+      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+      "question": "Does Activ Assure cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+      "expected_answer": "Yes",
+      "bot_answer": "Yes, Activ Assure Diamond does cover AYUSH treatments (Ayurveda, Yoga, Unani, Siddha, Homeopathy) under specific conditions. The policy reimburses in-patient hospitalization expenses for Ayush treatments conducted at recognized institutions [Source: Activ Assure Diamond (aditya-birla), p.3-4].  \n\nKey requirements:  \n1. Treatment must be at government hospitals, CCIM/CCH-recognized AYUSH colleges, or accredited institutions with \u226515 beds and qualified doctors  \n2. Must be performed by a registered AYUSH practitioner  \n3. Only covers in-patient expenses (excludes pre/post-hospitalization costs)  \n\nNote: This coverage is subject to policy schedule limits and hospital accreditation criteria. For out-patient Ayush consultations, coverage isn't mentioned in the provided clauses.",
+      "factual_match": true,
+      "citation_present": true,
+      "judge_score": 1.0,
+      "judge_reason": "BOT accurately conveys the information",
+      "expected_refusal": false,
+      "question_type": "coverage_scope",
+      "difficulty": "easy",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 9438
+    },
+    {
+      "id": "aditya-birla__activ-assure-diamond__wordings::room_rent_capping::medium",
+      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+      "question": "Is there a cap on room rent under Activ Assure?",
+      "expected_answer": "Single Private A/C Room (upgradable to next level, only if Single Private A/C Room is not available)",
+      "bot_answer": "The room rent cap under Activ Assure is specified in your Policy Schedule/Product Benefit Table, as per the sub-limits mentioned [Source: Activ Assure Diamond (aditya-birla), p.1]. For example, a \"Single Private A/C Room\" is defined as the most economical accommodation category with specific limits [Source: Activ Assure Diamond (aditya-birla), p.23-24].  \n\nNote: This is subject to IRDAI regulations mandating reasonable sub-limits. For your exact cap, check the \"Room Rent\" section in your Policy Schedule or contact Aditya Birla Health directly.",
+      "factual_match": false,
+      "citation_present": true,
+      "judge_score": 0.0,
+      "judge_reason": "BOT does not convey the upgradability to the next level.",
+      "expected_refusal": false,
+      "question_type": "sub_limit",
+      "difficulty": "medium",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 9504
+    },
+    {
+      "id": "aditya-birla__activ-assure-diamond__wordings::REFUSE::exclusions_oos::hard",
+      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+      "question": "Does Activ Assure cover injuries from space tourism?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "Activ Assure Diamond does not explicitly cover injuries from space tourism. The policy excludes \"participation or involvement in... professional or semiprofessional nature\" activities like aviation, scuba diving, or paragliding [Source: Activ Assure Diamond (aditya-birla), p.13]. Space tourism would likely be considered a high-risk activity falling under this exclusion. Additionally, injuries from \"willful or deliberate exposure to danger\" are not covered [Same source].  \n\nNote: This policy has a 36-month waiting period for pre-existing conditions, but space tourism exclusions apply regardless. For clarity, I recommend confirming with the insurer directly before purchasing, as space tourism is not a standard coverage scenario.",
+      "factual_match": false,
+      "citation_present": true,
+      "judge_score": 0.0,
+      "judge_reason": "did not refuse when expected",
+      "expected_refusal": true,
+      "question_type": "exclusions_oos",
+      "difficulty": "hard",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 9966
+    },
+    {
+      "id": "aditya-birla__activ-assure-diamond__wordings::REFUSE::exclusions_oos::hard",
+      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+      "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Activ Assure?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": true,
+      "citation_present": false,
+      "judge_score": 1.0,
+      "judge_reason": "correctly refused",
+      "expected_refusal": true,
+      "question_type": "exclusions_oos",
+      "difficulty": "hard",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10515
+    },
+    {
+      "id": "aditya-birla__activ-assure-diamond__wordings::REFUSE::regulatory_oos::hard",
+      "policy_id": "aditya-birla__activ-assure-diamond__wordings",
+      "question": "What is the IRDAI mandate on dental coverage that Activ Assure must follow?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": true,
+      "citation_present": false,
+      "judge_score": 1.0,
+      "judge_reason": "correctly refused",
+      "expected_refusal": true,
+      "question_type": "regulatory_oos",
+      "difficulty": "hard",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10159
+    },
+    {
+      "id": "bajaj-allianz__comprehensive-care-plan__wordings::pre_existing_disease_waiting_months::easy",
+      "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+      "question": "What is the waiting period for pre-existing diseases under Comprehensive Care Plan?",
+      "expected_answer": "36 months from policy inception",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "easy",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 13850
+    },
+    {
+      "id": "bajaj-allianz__comprehensive-care-plan__wordings::pre_existing_disease_waiting_months::medium",
+      "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+      "question": "If I have diabetes, how long do I have to wait before I can claim under Comprehensive Care Plan?",
+      "expected_answer": "36 months \u2014 pre-existing diseases have a waiting period of 36 months from policy start",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "medium",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 11087
+    },
+    {
+      "id": "bajaj-allianz__comprehensive-care-plan__wordings::initial_waiting_period_days::easy",
+      "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+      "question": "What is the initial waiting period under Comprehensive Care Plan?",
+      "expected_answer": "90 days from policy inception",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "easy",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 11584
+    },
+    {
+      "id": "bajaj-allianz__comprehensive-care-plan__wordings::ayush_coverage::easy",
+      "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+      "question": "Does Comprehensive Care Plan cover AYUSH (Ayurveda, Yoga, Unani, Siddha, Homeopathy)?",
+      "expected_answer": "Yes",
+      "bot_answer": "The Comprehensive Care Plan does cover AYUSH treatments (Ayurveda, Yoga, Unani, Siddha, Homeopathy) **if** the treatment is conducted in a recognized AYUSH Hospital or Day Care Centre as defined in the policy. Coverage includes hospitalization and day-care procedures under these systems [Source: Comprehensive Care Plan (bajaj-allianz), p.1-2].  \n\n**Key criteria for AYUSH facilities**:  \n- Minimum 5 in-patient beds (15 in larger towns)  \n- 24/7 qualified AYUSH practitioners  \n- Dedicated therapy sections/operation theatres  \n- Registered with local authorities [Source: Comprehensive Care Plan (bajaj-allianz), p.1-2].  \n\nNote: This coverage applies only to in-patient hospitalization or day-care procedures at these facilities. For out-patient AYUSH consultations, the policy does not explicitly mention coverage in the provided clauses.",
+      "factual_match": true,
+      "citation_present": true,
+      "judge_score": 1.0,
+      "judge_reason": "BOT accurately conveys the coverage of AYUSH treatments.",
+      "expected_refusal": false,
+      "question_type": "coverage_scope",
+      "difficulty": "easy",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10873
+    },
+    {
+      "id": "bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::exclusions_oos::hard",
+      "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+      "question": "Does Comprehensive Care Plan cover injuries from space tourism?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": true,
+      "citation_present": false,
+      "judge_score": 1.0,
+      "judge_reason": "correctly refused",
+      "expected_refusal": true,
+      "question_type": "exclusions_oos",
+      "difficulty": "hard",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "fabricated_citation: same as above, clause 3"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 9331
+    },
+    {
+      "id": "bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::exclusions_oos::hard",
+      "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+      "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Comprehensive Care Plan?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "The Comprehensive Care Plan policy document provided does not mention any specific coverage limits or exclusions for diamond-tipped surgical procedures. The retrieved clauses focus on general terms, claim timelines, exclusions like terrorism, and benefits for critical illnesses/accidents, but no procedure-specific limits are indicated [Source: Comprehensive Care Plan (bajaj-allianz), pp.1-16].\n\nFor clarity, the policy covers surgical procedures under Section 5 (Critical Illness) and Section 2 (Accident Protection), but these are subject to general terms like waiting periods and exclusions. Since the document lacks details about diamond-tipped procedures, I cannot confirm coverage for this specific case.\n\nNote: This type of specialized procedure might require pre-authorization. I\u2019d recommend you confirm with the insurer directly before finalizing any treatment.",
+      "factual_match": false,
+      "citation_present": true,
+      "judge_score": 0.0,
+      "judge_reason": "did not refuse when expected",
+      "expected_refusal": true,
+      "question_type": "exclusions_oos",
+      "difficulty": "hard",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 9925
+    },
+    {
+      "id": "bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::regulatory_oos::hard",
+      "policy_id": "bajaj-allianz__comprehensive-care-plan__wordings",
+      "question": "What is the IRDAI mandate on dental coverage that Comprehensive Care Plan must follow?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "The Comprehensive Care Plan policy document does not explicitly mention any IRDAI mandates specifically for dental coverage. The retrieved clauses focus on general hospitalization, AYUSH facilities, and day care centers without dental-specific provisions [Source: Comprehensive Care Plan (bajaj-allianz), pp.1-30].  \n\nFor dental coverage details, you would need to check the policy's excluded treatments or consult IRDAI's general health insurance regulations (not provided in this context). I recommend confirming directly with Bajaj Allianz or reviewing the policy's exclusions section for dental-related clauses.",
+      "factual_match": false,
+      "citation_present": true,
+      "judge_score": 0.0,
+      "judge_reason": "did not refuse when expected",
+      "expected_refusal": true,
+      "question_type": "regulatory_oos",
+      "difficulty": "hard",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10360
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::pre_existing_disease_waiting_months::easy",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "What is the waiting period for pre-existing diseases under Silver Health?",
+      "expected_answer": "24 months from policy inception",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "easy",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 15886
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::pre_existing_disease_waiting_months::medium",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "If I have diabetes, how long do I have to wait before I can claim under Silver Health?",
+      "expected_answer": "24 months \u2014 pre-existing diseases have a waiting period of 24 months from policy start",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "medium",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 12098
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::initial_waiting_period_days::easy",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "What is the initial waiting period under Silver Health?",
+      "expected_answer": "30 days from policy inception",
+      "bot_answer": "The initial waiting period under Silver Health (Bajaj Allianz) is **30 days for all illnesses** as per the policy clauses [Source: Silver Health (bajaj-allianz), pp.2-4]. This applies to all covered treatments except specific procedures listed under the 12-month waiting period. Note: This is the policy's standard waiting period \u2013 confirm with the insurer if your specific condition/procedure has additional waiting periods.",
+      "factual_match": false,
+      "citation_present": true,
+      "judge_score": 0.0,
+      "judge_reason": "judge_error: HTTPStatusError: Client error '429 Too Many Requests' for url 'https://api.groq.com/openai/v1/chat/completions'\nFor more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
+      "expected_refusal": false,
+      "question_type": "waiting_period",
+      "difficulty": "easy",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 11012
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::pre_hospitalization_days::easy",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "How many days of pre-hospitalization expenses does Silver Health cover?",
+      "expected_answer": "30 days",
+      "bot_answer": "Silver Health covers **30 days** of pre-hospitalization expenses prior to hospitalization, as per the policy clauses [Source: Silver Health (bajaj-allianz), p.1].  \n\nNote: This is subject to the general terms of the policy and any applicable exclusions. For example, expenses related to investigations or non-medical reasons (as listed in exclusions) would not be covered.",
+      "factual_match": true,
+      "citation_present": true,
+      "judge_score": 1.0,
+      "judge_reason": "BOT accurately conveys the key information.",
+      "expected_refusal": false,
+      "question_type": "coverage_scope",
+      "difficulty": "easy",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 8947
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::post_hospitalization_days::easy",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "How many days of post-hospitalization expenses does Silver Health cover?",
+      "expected_answer": "60 days",
+      "bot_answer": "Silver Health covers post-hospitalization expenses for **up to 60 days** from the date of discharge, as per the policy clauses [Source: Silver Health (bajaj-allianz), p.1].  \n\nNote: This is subject to the insured being admitted for over 24 hours. For clarity, you may confirm this with the insurer directly before finalizing.",
+      "factual_match": true,
+      "citation_present": true,
+      "judge_score": 1.0,
+      "judge_reason": "BOT accurately conveys the post-hospitalization coverage period.",
+      "expected_refusal": false,
+      "question_type": "coverage_scope",
+      "difficulty": "easy",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 9620
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::no_claim_bonus_pct::easy",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "What's the no-claim bonus on Silver Health?",
+      "expected_answer": "10.0% step-up on sum insured per claim-free year",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "bonus",
+      "difficulty": "easy",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "unsupported_number: 2 years"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 9797
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::room_rent_capping::medium",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "Is there a cap on room rent under Silver Health?",
+      "expected_answer": "1% of hospitalization Sum Insured up to maximum Rs. 7,500 per day",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": false,
+      "citation_present": false,
+      "judge_score": 0.0,
+      "judge_reason": "bot refused on a question with a known answer",
+      "expected_refusal": false,
+      "question_type": "sub_limit",
+      "difficulty": "medium",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "unsupported_number: \u20b92",
+        "unsupported_number: \u20b92"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10416
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::copayment_pct::easy",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "Is there a copayment under Silver Health?",
+      "expected_answer": "10% copayment applies",
+      "bot_answer": "Silver Health (bajaj-allianz) does not have a mandatory copayment clause in the base policy. However, specific add-ons like **cataract coverage** require a 10% copayment (max \u20b940,000 per claim) [Source: Silver Health (bajaj-allianz), p.3]. For non-network hospitals, a 20% copayment applies per claim unless an extra premium is paid to waive it [Source: Silver Health (bajaj-allianz), p.3].  \n\nNote: This varies by plan (A/B) and add-ons selected.",
+      "factual_match": true,
+      "citation_present": true,
+      "judge_score": 1.0,
+      "judge_reason": "BOT accurately conveys the copayment information.",
+      "expected_refusal": false,
+      "question_type": "sub_limit",
+      "difficulty": "easy",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10675
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::REFUSE::exclusions_oos::hard",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "Does Silver Health cover injuries from space tourism?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "Silver Health by Bajaj Allianz does not explicitly cover injuries from space tourism based on the provided policy clauses. The policy covers in-patient hospitalization for accidental bodily injuries (Section C1) but excludes treatments outside India (Exclusion DII) and does not mention space tourism as a covered scenario.  \n\n**Key reasons:**  \n1. **Geographic exclusion**: All treatments outside India are excluded [Source: Silver Health (bajaj-allianz), p.2-4, Exclusion DII]. Space tourism typically occurs outside India.  \n2. **No specific coverage**: The policy does not list space tourism or high-altitude/aviation-related injuries as covered risks.  \n\n**Note**: This policy has a 30-day initial waiting period for all illnesses [Source: Silver Health (bajaj-allianz), p.3-5, Standard Time Period]. Even if coverage were claimed, pre-existing conditions related to such activities would face a 12-month waiting period.  \n\nI recommend you confirm directly with the insurer, as non-standard activities often require specific riders or exclusions.",
+      "factual_match": false,
+      "citation_present": true,
+      "judge_score": 0.0,
+      "judge_reason": "did not refuse when expected",
+      "expected_refusal": true,
+      "question_type": "exclusions_oos",
+      "difficulty": "hard",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10754
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::REFUSE::exclusions_oos::hard",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "What is the maximum claim amount for diamond-tipped surgical procedures under Silver Health?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "The provided policy documents do not specifically mention diamond-tipped surgical procedures under covered treatments. While the policy covers advanced procedures like robotic surgeries and stereotactic radio surgeries [Source: Silver Health (bajaj-allianz), pp.2-3], there is no explicit reference to \"diamond-tipped\" procedures in the listed inclusions or exclusions.  \n\nFor clarity:  \n1. If the procedure falls under the broader category of \"Modern Treatment Methods\" (e.g., robotic surgeries), it *may* be covered up to the sum insured [Source: Silver Health (bajaj-allianz), pp.2-3].  \n2. If it is a cosmetic or elective procedure, it would likely be excluded under Section D II (cosmetic surgery) or Exclusion 10 (non-medical items) [Source: Silver Health (bajaj-allianz), pp.2-4].  \n\n**Note**: The policy does not specify sub-limits for individual procedures beyond the overall sum insured. For confirmation, you would need to share the specific procedure details with the insurer, as coverage depends on whether it aligns with the defined categories.",
+      "factual_match": false,
+      "citation_present": true,
+      "judge_score": 0.0,
+      "judge_reason": "did not refuse when expected",
+      "expected_refusal": true,
+      "question_type": "exclusions_oos",
+      "difficulty": "hard",
+      "blocked": false,
+      "faithfulness_passed": true,
+      "faithfulness_reasons": [],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 11775
+    },
+    {
+      "id": "bajaj-allianz__silver-health__cis::REFUSE::regulatory_oos::hard",
+      "policy_id": "bajaj-allianz__silver-health__cis",
+      "question": "What is the IRDAI mandate on dental coverage that Silver Health must follow?",
+      "expected_answer": "Bot should refuse or say not in document.",
+      "bot_answer": "I'd rather not answer that without stronger evidence in the policy documents I have. Could you rephrase, or narrow your question to a specific policy?",
+      "factual_match": true,
+      "citation_present": false,
+      "judge_score": 1.0,
+      "judge_reason": "correctly refused",
+      "expected_refusal": true,
+      "question_type": "regulatory_oos",
+      "difficulty": "hard",
+      "blocked": true,
+      "faithfulness_passed": false,
+      "faithfulness_reasons": [
+        "gate4_llm_judge: claims unsupported"
+      ],
+      "brain_used": "sarvam-m::simple-qa",
+      "latency_ms": 10545
+    }
+  ]
+}

eval/results.md ADDED Viewed

	@@ -0,0 +1,55 @@

+# Eval Results — 2026-05-12T22:30:15Z
+## Headline
+| Metric | Value |
+| --- | --- |
+| Questions run | 25 |
+| **Factual accuracy** | **40.0%** |
+| **Citation accuracy** | **50.0%** |
+| **Refusal precision** | **44.4%** |
+| Blocked by faithfulness | 12 |
+| Elapsed | 293.0 s |
+## By question type
+| Type | Accuracy |
+| --- | --- |
+| coverage_scope | 100.0% |
+| regulatory_oos | 66.7% |
+| sub_limit | 33.3% |
+| exclusions_oos | 33.3% |
+| waiting_period | 12.5% |
+| bonus | 0.0% |
+## By brain (router winners)
+| Brain | Accuracy |
+| --- | --- |
+| groq-llama | 100.0% |
+| sarvam-m | 37.5% |
+## Sample misses (up to 15)
+| id | question | bot_answer | reason |
+| --- | --- | --- | --- |
+| aditya-birla__activ-assure-diamond__wordings::pre_existing_d | If I have diabetes, how long do I have to wait before I can  | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| aditya-birla__activ-assure-diamond__wordings::room_rent_capp | Is there a cap on room rent under Activ Assure? | The room rent cap under Activ Assure is specified in your Policy Schedule/Produc | BOT does not convey the upgradability to the next level. |
+| aditya-birla__activ-assure-diamond__wordings::REFUSE::exclus | Does Activ Assure cover injuries from space tourism? | Activ Assure Diamond does not explicitly cover injuries from space tourism. The  | did not refuse when expected |
+| bajaj-allianz__comprehensive-care-plan__wordings::pre_existi | What is the waiting period for pre-existing diseases under C | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| bajaj-allianz__comprehensive-care-plan__wordings::pre_existi | If I have diabetes, how long do I have to wait before I can  | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| bajaj-allianz__comprehensive-care-plan__wordings::initial_wa | What is the initial waiting period under Comprehensive Care  | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::ex | What is the maximum claim amount for diamond-tipped surgical | The Comprehensive Care Plan policy document provided does not mention any specif | did not refuse when expected |
+| bajaj-allianz__comprehensive-care-plan__wordings::REFUSE::re | What is the IRDAI mandate on dental coverage that Comprehens | The Comprehensive Care Plan policy document does not explicitly mention any IRDA | did not refuse when expected |
+| bajaj-allianz__silver-health__cis::pre_existing_disease_wait | What is the waiting period for pre-existing diseases under S | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| bajaj-allianz__silver-health__cis::pre_existing_disease_wait | If I have diabetes, how long do I have to wait before I can  | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| bajaj-allianz__silver-health__cis::initial_waiting_period_da | What is the initial waiting period under Silver Health? | The initial waiting period under Silver Health (Bajaj Allianz) is **30 days for  | judge_error: HTTPStatusError: Client error '429 Too Many Req |
+| bajaj-allianz__silver-health__cis::no_claim_bonus_pct::easy | What's the no-claim bonus on Silver Health? | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| bajaj-allianz__silver-health__cis::room_rent_capping::medium | Is there a cap on room rent under Silver Health? | I'd rather not answer that without stronger evidence in the policy documents I h | bot refused on a question with a known answer |
+| bajaj-allianz__silver-health__cis::REFUSE::exclusions_oos::h | Does Silver Health cover injuries from space tourism? | Silver Health by Bajaj Allianz does not explicitly cover injuries from space tou | did not refuse when expected |
+| bajaj-allianz__silver-health__cis::REFUSE::exclusions_oos::h | What is the maximum claim amount for diamond-tipped surgical | The provided policy documents do not specifically mention diamond-tipped surgica | did not refuse when expected |
+---
+*Grader: Groq Llama-3.3-70B-versatile (different model family from Sarvam-M to avoid circular eval).*
+*Full per-question results: `eval/results.json`.*

rag/vectors/chroma.sqlite3 CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b76b74613c526a44e9fd30fec2c7b4b2a126c585c62f4ea6f8e4ba5d79ddc5e
 size 72695808

 version https://git-lfs.github.com/spec/v1
+oid sha256:046dda5d699f5975831d6ee8ab1e0864c4bbd73096302ad271610140233e86ee
 size 72695808

tools/set_hf_secrets.py ADDED Viewed

	@@ -0,0 +1,58 @@

+"""Programmatically set HF Space secrets via the official HF API.
+No browser needed. Reads keys from local .env and pushes them as Space
+secrets so the deployed app can authenticate to Sarvam / Voyage / Groq / OpenRouter.
+Run:
+  python tools/set_hf_secrets.py
+"""
+from __future__ import annotations
+import os
+import sys
+from pathlib import Path
+from dotenv import load_dotenv
+from huggingface_hub import HfApi
+ROOT = Path(__file__).resolve().parent.parent
+load_dotenv(ROOT / ".env")
+REPO_ID = "rohitsar567/InsuranceBot"
+SECRETS_TO_SET = ["SARVAM_API_KEY", "VOYAGE_API_KEY", "GROQ_API_KEY", "OPENROUTER_API_KEY"]
+def main():
+    hf_token = os.environ.get("HF_TOKEN")
+    if not hf_token:
+        print("ERROR: HF_TOKEN missing in .env")
+        return 1
+    api = HfApi(token=hf_token)
+    print(f"Setting {len(SECRETS_TO_SET)} secrets on {REPO_ID}...")
+    for key in SECRETS_TO_SET:
+        value = os.environ.get(key)
+        if not value:
+            print(f"  - {key}: MISSING in local .env — skipping")
+            continue
+        try:
+            api.add_space_secret(
+                repo_id=REPO_ID,
+                key=key,
+                value=value,
+                description=f"Set programmatically on {key}",
+            )
+            print(f"  - {key}: SET (length={len(value)})")
+        except Exception as e:
+            print(f"  - {key}: FAIL {type(e).__name__}: {e}")
+    print()
+    print("Done. The Space will rebuild automatically.")
+    print(f"Watch progress: https://huggingface.co/spaces/{REPO_ID}?logs=build")
+    return 0
+if __name__ == "__main__":
+    sys.exit(main())