update evaluation

eval_pipeline.py

@@ -38,11 +38,6 @@ retriever = vectorstore.as_retriever(
 # test_docs_scores = vectorstore.similarity_search_with_score("what are the fees", k=3)
 # print("With scores:", test_docs_scores)
 
-from difflib import SequenceMatcher
-
-# def is_correct(predicted: str, expected: str, threshold=0.6) -> bool:
-#     ratio = SequenceMatcher(None, predicted.lower(), expected.lower()).ratio()
-#     return ratio >= threshold
 
 def llm_judge(query, expected, predicted, llm):
     prompt = f"""
@@ -136,111 +131,6 @@ def retrieval_success_check(expected, retrieved_text, threshold=0.6):
 #     for _, row in test_df.iterrows()
 # ]
 
-# def evaluate(dataset: list[dict], split: str = "", sample: int = None) -> float:
-#     data = random.sample(dataset, sample) if sample else dataset
-#     correct = 0
-#     for item in tqdm(data, desc=f"Evaluating {split}", unit="q"):
-#         predicted = ask(query=item["query"], persist=False)
-#         if is_correct(predicted, item["expected_answer"]):
-#             correct += 1
-#     return correct / len(data)
-
-# train_accuracy = evaluate(train_dataset, split="train", sample=200)
-# test_accuracy  = evaluate(test_dataset,  split="test",  sample=100)
-
-# print(f"Train accuracy: {train_accuracy:.2%}")
-# print(f"Test accuracy:  {test_accuracy:.2%}")
-
-# import matplotlib.pyplot as plt
-# import seaborn as sns 
-# labels     = ["Train", "Test"]
-# accuracies = [train_accuracy, test_accuracy]
-
-# plt.figure(figsize=(6, 4))
-# plt.bar(labels, accuracies, color=["steelblue", "coral"], width=0.4)
-# plt.ylim(0, 1)
-# plt.ylabel("Accuracy")
-# plt.title("Chatbot Accuracy — Train vs Test")
-
-# for i, v in enumerate(accuracies):
-#     plt.text(i, v + 0.02, f"{v:.1%}", ha="center", fontweight="bold")
-
-# plt.tight_layout()
-# plt.savefig("accuracy.png")
-# plt.show()
-
-
-# def evaluate(dataset: list[dict], split: str = "", sample: int = None):
-#     data = random.sample(dataset, sample) if sample else dataset
-
-#     results = []
-
-#     correct           = 0
-#     hallucinated      = 0
-#     retrieval_failures = 0
-#     total_recall      = 0
-#     total_precision   = 0
-
-#     for item in tqdm(data, desc=f"Evaluating {split}", unit="q"):
-
-#         query              = item["query"]
-#         expected           = item["expected_answer"]
-#         relevant_doc_ids   = item["relevant_doc_ids"]   # ← from your tagged dataset
-
-#         # run chatbot
-#         predicted = ask(query=query, persist=False)
-
-#         # --- retrieval ---
-#         retrieved_docs = retriever.invoke(query)
-#         recall, precision = retrieval_metrics(retrieved_docs, relevant_doc_ids, k=K)
-
-#         retrieval_success = recall > 0   # at least one relevant chunk retrieved
-
-#         total_recall    += recall
-#         total_precision += precision
-
-#         if not retrieval_success:
-#             retrieval_failures += 1
-
-#         # --- correctness ---
-#         correct_flag = is_correct(predicted, expected)
-
-#         # --- hallucination ---
-#         # hallucination = retrieval worked but LLM still got it wrong
-#         # (if retrieval failed, it's a retrieval problem, not hallucination)
-#         hallucinated_flag = retrieval_success and not correct_flag
-
-#         if correct_flag:
-#             correct += 1
-
-#         if hallucinated_flag:
-#             hallucinated += 1
-
-#         results.append({
-#             "query":            query,
-#             "expected":         expected,
-#             "predicted":        predicted,
-#             "relevant_doc_ids": relevant_doc_ids,
-#             "retrieved_doc_ids": [doc.metadata["doc_id"] for doc in retrieved_docs[:K]],
-#             "retrieval_success": retrieval_success,
-#             f"recall@{K}":      recall,
-#             f"precision@{K}":   precision,
-#             "correct":          correct_flag,
-#             "hallucinated":     hallucinated_flag
-#         })
-
-#     metrics = {
-#         "split":                 split,
-#         "accuracy":              correct / len(data),
-#         "hallucination_rate":    hallucinated / len(data),
-#         "retrieval_failure_rate": retrieval_failures / len(data),
-#         f"recall@{K}":           total_recall / len(data),
-#         f"precision@{K}":        total_precision / len(data),
-#         "samples":               len(data)
-#     }
-
-#     return metrics, results
-
 def measure_hallucination(query: str, response: str, chunks: list[dict], llm) -> dict:
     """
     Uses the LLM to check whether the response contains claims

test_eval_v2.json

@@ -0,0 +1,811 @@
+{
+  "metrics": {
+    "split": "test",
+    "accuracy": 0.23,
+    "hallucination_rate": 0.77,
+    "retrieval_failure_rate": 1.0,
+    "samples": 100
+  },
+  "detailed_results": [
+    {
+      "query": "Do I have to pay any margin money - my own funding (Education Loans For Indian Education)",
+      "expected": "Margin Money upto Rs. 4.00 lacs - Nil, For loans above Rs. 4.00 lacs - 5%",
+      "predicted": "Margin Money upto Rs. 4.00 lacs - Nil, For loans above Rs. 4.00 lacs - 15%.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What do you mean by pre-existing diseases not covered under the Critical Illness plan",
+      "expected": "A pre-existing disease is any condition, ailment, injury or related condition for which the insured person had signs or symptoms and/or was diagnosed and/or received medical advice or treatment prior to the first policy with the company",
+      "predicted": "A pre-existing disease is any condition, ailment, injury or related condition for which the insured person had signs or symptoms and/or was diagnosed and/or received medical advice or treatment prior to the first policy with the company.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "Does M&N Bank offer preferential rate of interest on Fixed Deposits for Senior Citizens",
+      "expected": "We offer preferential rate of interest on Fixed Deposits for Senior Citizens. If you are a senior citizen, all you need to do is register as one by calling any Branch with proof of your Date of Birth. Once registered as a Senior Citizen, you are eligible for preferential rates of interest on Fixed Deposits opened through NetBanking as well. Click here to Interest Rates on Fixed Deposits. The Fixed Deposit will be booked in the same combination/ order of names as the existing Current/Savings account, from which the funds are being transferred. View more",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "During the policy period, if I add accessories like a music system in my vehicle, are they also covered",
+      "expected": "Yes, they will be, as long as you make a written request to us with a description of the accessory along with its value. You may also have to pay an additional premium.",
+      "predicted": "Yes, additions such as accessories (e.g. a music system) are guaranteed to be paid for every completed policy term.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the premium frequency available under policy",
+      "expected": "The premiums can be paid annually, half-yearly, quarterly and annually.",
+      "predicted": "The premiums can be paid Annually, Half-Yearly, Quarterly and Monthly.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "How much can I borrow",
+      "expected": "You can take a loan ranging from Rs. 50,000 to Rs. 15,00,000, depending on your income and repayment capacity. You can club your spouses income if you want to get a bigger loan.",
+      "predicted": "You can avail of a loan from Rs 50,000/- to Rs 50,00,000/- depending on your income, repayment capacity and respective location product cap.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What is the minimum balance required for an Institutional Current Account",
+      "expected": "No minimum balance requirement.",
+      "predicted": "The minimum balance requirement for an Institutional Current Account is not explicitly mentioned in the provided context. However, it's possible that there might be a specific average quarterly balance (AQB) or other requirements mentioned elsewhere in the bank's policies or documentation. If you could provide more information or clarify what you're looking for, I'd be happy to try and help further.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How can I change / add the email ID in my account",
+      "expected": "We request you to follow the procedure mentioned below to change your email ID:We request you to follow either of procedure mentioned below to change / add your email ID: 1) Request through branch:Step 1) Fill the application form available at the Branch (duly signed by all the holders). Step 2) If you are unable to visit an M&N Bank Branch, you can send the Form by post or courier. Note / Important: Instructions received through Net Banking, scan copy, fax or email will not be processed for security and authentication purposes. Updates for InstaAlerts (if registered) need to be done separately. InstaAlerts Updates: Please follow the steps mentioned below to update your Email ID online for InstaAlerts (in case you are registered for the InstaAlerts facility): Step 1) Log in to your NetBanking and click on InstaAlerts located at the top right corner of the web page Step 2) Click on Unsubscribe Alerts on the left hand pane of the web page Step 3) Check the Email option and select Continue and then Confirm Step 4) Click on Subscribe for Alerts in the left hand pane of the web page Step 5) Check the Email option and enter your new mobile number, then click on Continue and Confirm Step 6) Go to Manage Alerts and select the required option 2) Request through Phone Banking:We request you to call up our PhoneBanking to change / add your email address. View more",
+      "predicted": "We request you get in touch with the nearest branch to enable us to register you for E-mail Statements of your Mutual Funds. On receipt of the form, your e-mail ID will be updated with the respective AMC within 15 working days. You will receive the holding statements on your email and mailing address simultaneously after registration.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What are the available plans under the Critical Illness policy",
+      "expected": "You can choose to be insured for any of the following amounts: Rs. 2,50,000, Rs. 5,00,000, Rs.7,50,000 or Rs. 10,00,000.",
+      "predicted": "The available plans under the Critical Illness policy are:\n\n1. Critical Illness - Platinum Plan\n2. View more (this suggests that there may be other plans available, but it's not specified what they are)",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is M&N Bank ForexPlus Chip card",
+      "expected": "M&N Bank ForexPlus Chip card is a magnetic strip based prepaid card introduced in partnership with Visa / Mastercard International. The card offers you a convenient and secure way to carry forex. This card offers you the freedom to shop anywhere in the world and provides access to the currency of your destination at competitive market rates. It can be used at all Visa / Mastercard ATMs and Merchant establishment worldwide. This card cannot be used for making payments in India, Nepal or Bhutan.",
+      "predicted": "M&N Bank ForexPlus Platinum (Chip) card is an embedded chip based Prepaid card introduced in partnership with VISA International. It offers you a convenient and secure way to carry your Forex. This card gives you the freedom to shop anywhere in the world and provides access to the currency of your destination at competitive market rates. It can be used at all VISA ATMs and Merchant Establishments worldwide. This card cannot be used for payments in India, Nepal or Bhutan.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Who can open an Investment Services account",
+      "expected": "Any Individual or HUF, who has an M&N Bank Savings Account and is KYC compliant, can register for a Mutual Fund Investment Services Account. NRI customers who are KYC compliant and have a Savings account can also avail of this service. The Savings Bank account status has to be single or either/Survivor.",
+      "predicted": "Any Indian resident or Non-resident individual (not US-based) or Indian corporate can register with M&N Bank to open an Investment Services account, but currently, overseas corporate bodies (OCBs) cannot open an account directly with M&N Securities. To become a registered user of M&N Securities, you will have to first open a Savings/Current account and Demat account with M&N Bank.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can I avail an Overdraft against my Property",
+      "expected": "A Loan Against Property can be availed as a Term Loan with EMI repayments or as a Drop line Overdraft Facility.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "When does the Bank issue a TDS Certificate",
+      "expected": "The TDS Certificate, Form 16A, for TDS deducted during a calendar quarter, will be issued in the next month of the respective quarter.",
+      "predicted": "You will receive a consolidated TDS Certificate in Form 16A, for TDS deducted during a financial year, in the month of April of the following financial year.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Does the KYC Acknowledgement have an expiry date",
+      "expected": "No. Once the KYC Acknowledgement is obtained and informed to a Mutual Fund, it will be registered against the folio and quoted in all future account statements. The same will exist in perpetuity, unless cancelled by CVL.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can a Travel Insurance policy be extended",
+      "expected": "Extensions are applicable to original policies of 30 days or more and provided the policy is still in force.",
+      "predicted": "Yes, an extension under the policy shall be provided subject to certain conditions.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How many days in advance can I purchase foreign exchange",
+      "expected": "You can purchase foreign exchange up to 180 days in advance from the date of departure.",
+      "predicted": "You can purchase foreign exchange up to 60 days in advance from the date of departure.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "Whom should I contact if my Card is lost or stolen",
+      "expected": "Immediately report the loss of your M&N Bank Multi Currency ForexPlus Platinum Card by calling M&N Bank PhoneBanking. You will not be liable for transactions on your Card Account (if any) from the moment card loss is registered with the bank. You can also hotlist the card by using NetBanking services.",
+      "predicted": "Could you tell me more about what you're looking for? Are you referring to a specific person, like an account holder, and asking whom they should contact in case their card is lost or stolen?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "If I have just received a renewed card, do I need to re-register",
+      "expected": "No, you do not have to re-register your renewed card since the card number remains the same. You need to re-register only if your card number has changed due to card replacement/ re-issue/ upgrade.",
+      "predicted": "No, prior registration is not necessary for an OTP transaction.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How long is the tenure of the loan (Education Loans For Indian Education)",
+      "expected": "The maximum repayment tenures are as below: For loans upto Rs. 7.50 lacs - upto 10 years For loans above Rs. 7.50 lacs - upto 15 years",
+      "predicted": "The maximum tenure of the loan can be 7 years, including the Moratorium period.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Which cars can be financed for Loan against Car",
+      "expected": "Any Hatchbacks and Sedans, excluding some models that are out of production can be financed. To know more give us a call or visit a branch.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How long will it take to process the loan",
+      "expected": "The loan gets processed within 48 hours of completing documentation.",
+      "predicted": "Once you submit your application form and the required documents as per Bank's criteria, you can expect our approval and disbursal within 5 working days, provided everything is in order. All loan approvals are at the sole discretion of the bank.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the difference between 3D Life and 3D Life Long Protection options",
+      "expected": "In addition to the benefits under 3D Life Option, Life Assured is covered for whole life under 3D Life Long Protection option. Premiums under this option have to be paid only till an age of 65.",
+      "predicted": "The context doesn't explicitly state the differences between 3D Life and 3D Life Long Protection options, but it does mention that both have a Waiver of Premium benefit if the Life Assured is diagnosed with any of the 34 covered Critical Illnesses. However, I can infer from the context that there might be some additional features or benefits associated with each option.\n\nTo clarify, could you tell me more about what you're looking for in terms of differences between these two options?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What are the fund options",
+      "expected": "The plan offers 8 Funds (including 4 New Funds) of varying risk and return profiles. The Policyholder has the option to choose any one or more funds based on his/her Risk Appetite. The Fund Options available under the Plan are Equity Plus Fund, Diversified Equity Fund, BlueChip Fund, Opportunities Fund, Balanced Fund, Income Fund, Bond Fund and Conservative Fund",
+      "predicted": "The plan offers 4 funds of varying risk & return profiles. Income Fund, Balanced Fund, BlueChip Fund and Opportunities Fund",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What does 'Moratorium period' mean (Education Loans For Foreign Education)",
+      "expected": "A repayment moratorium (also called a repayment holiday) is the course work period + 1 year or 6 months after the student gets a job or starts earning, whichever is earlier.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How do I know my application has been accepted",
+      "expected": "You can check your account opening status on www.M&N sec.com on the footer page of the homepage section page under the Application status.",
+      "predicted": "According to our previous information, you will be informed once your application is accepted. However, it's not specified how this notification would be received. Could you tell me more about what you're looking for? Would you like to know if there's a specific contact method or timeframe for receiving the acceptance notification?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Do I have the option of pre-paying the entire loan amount",
+      "expected": "Yes. One can pre-pay the loan any time after 6 months of availing of the loan. One merely has to pay a small prepayment fee on the outstanding loan amount.",
+      "predicted": "Yes, you can pre-pay the loan any time after 6 months of availing of the loan. You just have to pay a small pre-payment fee on the outstanding loan amount.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "How do I obtain a NOC from the bank if I have lost my Registration Certificate book for a vehicle under finance with M&N Bank",
+      "expected": "You may obtain the NOC for a duplicate Registration Certificate book for your vehicle loan, by visiting any of our Retail Loan Service Center and submitting the following documents: Request form for applying for the required NOC. Self-attested copy of the Registration Certificate book, duly self-attested by you for validation. Copy of FIR or certificate from police or Acknowledgement from Police on written complaint with General Diary No. Applicable fee of Rs. 500/- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer,i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided.",
+      "predicted": "You may obtain the NOC for a duplicate Registration Certificate book for your vehicle loan, by visiting any of our Retail Loan Service Center and submitting the following documents: Request form for applying for the required NOC. Self attested copy of the lost Registration Certificate book, duly self attested by you, along with the Original Registration Certificate book for validation. Applicable fee of Rs. 500/- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer, i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What Property can be considered as collateral for LAP",
+      "expected": "You can provide your self-occupied Residential or Commercial property (Shops and Offices) as collateral.",
+      "predicted": "You can provide Residential or Commercial Property (Shops & Offices) leased to reputed lessees as collateral for a Loan Against Property (LAP).",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "How much time will it take to change the mode of repayment/ account for repayment of my loan",
+      "expected": "It will take 25 working days to activate the revised instructions. In case your next installment falls due prior to the said date, instruction already lodged with us will be presented for payment.",
+      "predicted": "It will take 25 working days to activate the revised instructions. In case your next instalment falls due prior to the 25 day period, instructions already lodged with us will be presented for payment.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What is the maximum value for which the Card can be used",
+      "expected": "There is no limit imposed on the usage of Hajj Umrah ForexPlus card. Customers can withdraw/utilise any amount up to the value loaded on Hajj Umrah ForexPlus card at any ATM/POS. However, there are some ATMs service providers which have put in per day / per transaction limit towards cash dispense which cannot be changed by M&N Bank. In such cases the cardholder might need to do multiple transactions if he wants to withdraw any amount higher than that allowed in one transaction.",
+      "predicted": "There is no limit imposed on the usage of ForexPlus Chip Card. Customer's can withdraw/utilise any amount, upto the value loaded on ForexPlus Chip card at any ATM/POS. However, there are some ATMs service providers which have put in per day / per transaction limit towards cash dispense which cannot be changed by the Bank. In such cases the cardholder might need to do multiple transactions if he wants to withdraw any amount higher than that allowed in one transaction.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "I do not have a Bank account with M&N Bank. Can I still apply through your Bank for IPOs using M&N Bank`s ASBA process",
+      "expected": "To apply for an IPO from M&N Bank using ASBA, you should have an account with the Bank. However, it is not compulsory for you to have a Demat Account.",
+      "predicted": "Could you tell me more about what you're looking for? Are you referring to a specific account type or product that would allow you to use M&N Bank's ASBA process, even if it's not with an M&N Bank account?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "FD",
+      "expected": "7%",
+      "predicted": "It seems like you're referring to a specific type of account or product. Are you referring to a Fixed Deposit (FD)?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How soon can I get an eGiftPlus Card",
+      "expected": "The beneficiary will get a real-time instant delivery of the M&N Bank eGiftPlus Card",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "I don't see my Personal Assurance Message on the Secure page . What should I do",
+      "expected": "Always look for your Personal Assurance Message before entering your Verified by Visa / MasterCard Securecode password. If your Personal Assurance Message does not appear on the secure page, do not enter your password. Please contact our Customer Care Center and update the same.",
+      "predicted": "Could you tell me more about what's happening when you try to access the Secure page? Are you getting an error message or is it just not appearing at all?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "In what transactions is the One Time Password (IVR Password ) process applicable",
+      "expected": "The IVR Password process is applicable only for Credit Cards for: Registration through M&N Bank website Registration during online shopping Password re-set through website Password re-set while shopping online View more",
+      "predicted": "The One Time Password (IVR Password) process is applicable for telephonic transactions made using your mobile or telephone.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the maximum value for which the Card can be used",
+      "expected": "There is no limit imposed on the usage of ForexPlus card. Customer's can withdraw/utilise any amount, upto the value loaded on card at any ATM/POS. However, there are some ATMs service providers which have put in per day / per transaction limit towards cash dispense which cannot be changed by the Bank. In such cases the cardholder might need to do multiple transactions if he wants to withdraw any amount higher than that allowed in one transaction.",
+      "predicted": "There is no limit imposed on the usage of ForexPlus Chip Card. Customer's can withdraw/utilise any amount, upto the value loaded on ForexPlus Chip card at any ATM/POS. However, there are some ATMs service providers which have put in per day / per transaction limit towards cash dispense which cannot be changed by the Bank. In such cases the cardholder might need to do multiple transactions if he wants to withdraw any amount higher than that allowed in one transaction.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What do you mean by pre-existing diseases",
+      "expected": "Any condition, ailment, injury of related condition(s) for which the insured person has had signs or symptoms and/or was diagnosed and/or has received medical advice/treatment within 48 months prior to the insureds first policy with the company.",
+      "predicted": "According to our policy terms, a pre-existing disease is any condition, ailment, injury or related condition for which the insured person had signs or symptoms and/or was diagnosed and/or received medical advice or treatment prior to the first policy with us.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What are the charges I have to pay to change the mode of repayment/ account for my loan",
+      "expected": "A service charge of Rs.500 is applicable for changes in loan repayment instructions issued from non-M&N Bank Account (ECS or PDC facility). A cheque/payorder/demand draft for Rs. 500 favouring M&N Bank Ltd Loan Account No. << >> will be required along with the revised set of repayment instructions.",
+      "predicted": "A service charge of Rs.500 is applicable for changes in loan repayment instructions issued from non-M&N Bank Account (ECS or PDC facility). A cheque/payorder/demand draft for Rs. 500 favouring M&N Bank Ltd Loan Account No. << >> will be required along with the revised set of repayment instructions",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What is the time frame to resolve Domestic ATM transaction disputes",
+      "expected": "7 working days from receipt of the dispute.",
+      "predicted": "Approximately 45 to 120 days from receipt of the dispute depending upon the relevant franchise's ( e.g. Visa / Master / Rupay / Diners etc.) dispute resolution cycle.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Who can open an RFC Domestic Account",
+      "expected": "Any person who is a resident Indian can open an RFC Domestic account, as long as the foreign exchange earned is in line with the permissible credits as defined in the RBI circular.",
+      "predicted": "An RFC Domestic Account can be opened by a resident individual.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Where can I use my EasyShop International Debit Card",
+      "expected": "Anywhere you see the Visa logo, and at any ATM displaying the Visa or PLUS logo. Remember, your card can be used at all M&N Bank ATMs just like a normal ATM card.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How do I top-up the value of my card when I am abroad",
+      "expected": "ForexPlus Chip card can also be loaded on your behalf by someone authorized by you. Authorised person has to visit M&N Bank branch with the necessary documents and funds. Well take care of the rest.",
+      "predicted": "ISIC M&N Bank ForexPlus Card can also be loaded on your behalf by someone authorized (blood relatives only) by you. Authorised person has to visit M&N Bank branch with the necessary documents and funds. Well take care of the rest. You can reload your ISIC ForexPlus Card multiple times in future till the card gets expired: You can authorize the Bank to debit your M&N Bank account by giving an M&N Bank cheque or make payment using a local cheque drawn on M&N Bank and reload the card for desired value. Reload requests can be made by any authorized person (blood relatives only) on your behalf. The person authorised by you for the same will need to come to bank branch and will need to provide the necessary documents to the bank to carry out reload. On receipt of the application form and the funds, your card will get reloaded for the desired amount. M&N Bank customers can also place a request to reload the card through M&N Bank PhoneBanking or M&N Bank NetBanking.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is pre-EMI interest",
+      "expected": "Pending final disbursement, you pay interest on the portion of the loan disbursed. This interest is called pre-EMI interest. Pre-EMI interest is payable every month from the date of each disbursement up to the date of commencement of EMI. *Conditions apply. All Home Loans from M&N Ltd. Credit at sole discretion of M&N Ltd.",
+      "predicted": "Could you tell me more about what you're looking for? Are you referring to a specific type of loan or product where pre-EMI interest applies?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How much is the Service Tax on Cross Currency Conversion, applicable at the time of Load, Reload and Refund",
+      "expected": "Following are the currency conversion service tax slabs applicable at the time of Load, Reload and Refund. For more details on other charges applicable on Regalia ForexPlus Card refer section Fees and Charges Buy and Sell Forex CurrencyService Tax Amount Up to Rs. 1 lakh 0.15% of the gross value or Rs. 37.5/- whichever is higher From Rs. 1 lakh to Rs 10 Lakhs Rs. 150 plus 0.075% of the amount exceeding Rs. 1 lakh From Rs. 10 lakhs and above Rs. 825 plus 0.015% of the amount exceeding Rs. 10 Lakhs, subject to a maximum of Rs. 7500 (including Swachh Bharat Cess and Krishi Kalyan cess as applicable)",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "When do I start paying my EMI",
+      "expected": "Credilas unique education loan offers the flexibility of paying only simple interest (PEMI) during the course period and principle plus interest (EMI) after the completion of studies and grace period.",
+      "predicted": "You typically start paying your EMI after 6 months from the date of loan disbursal, as per the terms and conditions of your loan agreement. However, please check your loan agreement or contact us for specific details on your loan.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Why can't I have liquidity of complete amount on surrender",
+      "expected": "The plan aims to help in creating fund for retirement. Annuitisation of surrender benefit is mandated by the Regulator in the interest of the customer. Annuitisation ensures that the aim of the policy is not compromised for other financial needs.",
+      "predicted": "It seems that there is a specific reason why you cannot have liquidity of the complete amount on surrender. However, I'd like to clarify what you mean by \"complete amount\". Are you referring to the full policy value or something else?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the minimum balance that I need to maintain in a Max Current Account",
+      "expected": "You need to maintain an Average Quarterly Balance of Rs. 5,00,000 in your Max Current account.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What happens if my card is lost",
+      "expected": "In case your card gets lost or stolen, all you have to do is call M&N Bank PhoneBanking immediately and report the loss of your card. This facility is available 24 hrs on all days. The card can also be hotlisted using our Prepaid card NetBanking facility on www.M&N bank.com",
+      "predicted": "You can contact the below mentioned Customer Care number 022-33271010 or e-mail",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the M&N Bank Hajj Umrah ForexPlus card",
+      "expected": "The M&N Bank Hajj Umrah ForexPlus card is an embedded Chip based Prepaid Forex card introduced in partnership with VISA International. It is the first card exclusively designed for Hajj Umrah pilgrims to carry currency in a convenient and secure way. The card holders are also entitled for exclusive discounts & gift voucher at select merchant outlets in Saudi Arabia. The card can be used at all VISA ATMs and merchant establishments worldwide. This card cannot be used for payments in India, Nepal or Bhutan.",
+      "predicted": "The M&N Bank Hajj Umrah ForexPlus card is a prepaid forex card designed to facilitate easy and convenient transactions for pilgrims traveling to Saudi Arabia for Hajj or Umrah. It allows users to load foreign currency, which can be used to make payments at merchant establishments displaying the Visa symbol, withdraw cash from ATMs worldwide, and perform other international transactions.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Do I need a guarantor",
+      "expected": "No, but in case if you don't meet the Bank's norms then you may be required to have a guarantor surety. A guarantor becomes a co-endorser and assumes liability in event of default. To stand surety for your loan.",
+      "predicted": "No. The car itself acts as a security.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "When does the plan acquire Guaranteed Surrender value",
+      "expected": "The policy will acquire a Guaranteed Surrender Value (GSV) provided 2 full years premium has been paid for premium payment term of 7 years and 3 full years premium have been paid for premium payment term of 10 years or more.",
+      "predicted": "The policy will acquire a Guaranteed Surrender Value (GSV) provided 2 full years premium has been paid. The GSV shall be aggregate of: Percentage of total premiums paid Percentage of accrued bonuses and accrued guaranteed additions Please refer the sales brochure or contact your Relationship manager to know the percentage factors for GSV",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Where do I get this card",
+      "expected": "ForexPlus card is available over the counter at M&N Bank branches. You simply need to walk into our branch with required documentation and walk out with ForexPlus card. The card will get activated within 4 hours from realization of funds. Alternatively, the customer can apply for card through NetBanking login also.In case the customer applies the card through NetBanking the card will be delivered to the customer within 2 working days. This facility is available at select locations only.",
+      "predicted": "ISIC ForexPlus Card is available over the counter at M&N Bank branches. You simply need to walk into our branch with required documentation and walk out with ISIC ForexPlus Card. The card will get activated within 4 hours from realization of funds. Also you can apply by visiting www.M&N bank.com -> Apply Online section or www.M&N bank.com/personal/products/cards/prepaid-cards -> ISIC Student ID ForexPlus Card -> Buy Forex Card",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How many times of my annual salary can I borrow with M&N Bank Auto Loan",
+      "expected": "You can borrow up to 3 times your annual salary if you are a salaried professional and 6 times your annual income if you are a self-employed professional **. **This would also vary with higher loan value",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Are there charges I should know about International Debit Card",
+      "expected": "Another benefit of the EasyShop Platinum Debit card is that there are absolutely no transaction charges to when shopping at a merchant location. Fuel surcharge levied at select BPCL petrol pumps will be reversed within 75 days from the date of levy of the surcharge. Just make sure your card is swiped at M&N Banks terminal to avail petrol surcharge waiver. You can use your card charge-free at all M&N Bank ATMs, but remember to check the associated fees when using other bank ATMs. Please also remember that charges will be levied at Railway stations. This is as per industry practice. View more",
+      "predicted": "You can be sure that there are no transaction charges when using your card at a merchant location. There are also no petrol surcharges, or fees for using an M&N Bank ATM. Just remember to check for usage charge information if you access other bank ATMs.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "If I don't have an account with M&N Bank, can I still avail of Pre-owned Car Loans",
+      "expected": "Repayment is preferred from a M&N Bank account. Having an account with M&N Bank helps in better eligibility and approval rates.",
+      "predicted": "Yes, you can get a Pre-owned Car Loan from M&N Bank even if you don't have an account with us.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What should I do in the case of a void transaction for Titanium Royale Debit Card",
+      "expected": "Just fax us a copy of the void transaction slip. You can also send it to or submit the same at your nearest M&N Bank branch. For PhoneBanking numbers Click Here .",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the Policy Term available in this plan",
+      "expected": "Capital Shield provides a Policy Term of 10 years.",
+      "predicted": "Policy Term of 10 to 40 years is available under the plan.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can I surrender this policy",
+      "expected": "Yes you can surrender your policy. Once you surrender your policy, your risk cover will cease and surrender benefit will be paid out depending upon the year in which the policy is surrendered. Please refer to product brochure for more details.",
+      "predicted": "You can surrender your policy subject to a lock in period of 5 years. No discontinuance charge is charged in this policy. Please refer to product brochure for more details on discontinuance of premiums and surrender of policy.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What should I do in case I am unable to access my Company's Current Account through Net Banking",
+      "expected": "If you are unable to access your company account, it may be because you have entered an incorrect Customer Identification Number (Cust ID) to log into your NetBanking Account. While opening Accounts of Companies, separate Customer IDs are created in name of the Company as well as each of the Authorised Signatory(ies). The latter is distinct and has to be used to access NetBanking for the Company's Account. However, we regret our inability to disclose the appropriate Customer ID on an unsecure e-mail and request you to contact our nearest Branch or PhoneBanking Center for this.For more details, Contact Us",
+      "predicted": "If you are unable to access your company account, it may be because you have entered an incorrect Customer Identification Number (Cust ID) to log into your NetBanking Account. While opening Accounts of Companies, separate Customer IDs are created in name of the Company as well as each of the Authorized Signatory(ies). The latter is distinct and has to be used to access NetBanking for the Company's Account. However, we regret our inability to disclose the appropriate Customer ID on an unsecure e-mail and request you to contact our nearest Branch or PhoneBanking Center for this. For more details, Contact Us",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What is the Policy term/Premium payment term",
+      "expected": "Policy Term 10 Year Premium Payment Term Single",
+      "predicted": "The fixed policy term available under this product is 5 years, and you shall only pay premiums for the benefit(s) as long as the complete payout for that benefit has not happened.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Should I visit a PoS personally to obtain KYC Compliance",
+      "expected": "No. If you are not in a position to visit a PoS personally, the KYC Application Form along with the necessary documents (including originals if the copies are not attested) can be sent through the distributor or representative who can arrange to complete the KYC process and obtain the KYC Acknowledgement through any PoS.",
+      "predicted": "Currently, KYC is done for free of cost.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Do I need to pay additional amount over and above Cash withdrawal Charges levied by the M&N Bank for ATM transaction",
+      "expected": "In certain countries ATM Service providers are authorized to charge surcharge/convenience fees from the cardholders for accessing the ATM to withdraw money. These charges are levied by the ATM service provider over and above the charges levied by the M&N Bank, and M&N Bank bank has no control over the charges as these charges are not levied by the bank.These charges are normally displayed to the customer during the transaction and the customer has the option to either proceed with the transaction or abort if the charges are not acceptable to the customer.",
+      "predicted": "In certain countries, ATM Service providers are authorized to charge surcharge/convenience fees from the cardholders for accessing the ATM to withdraw money. These charges are levied by the ATM service provider over and above the charges levied by the M&N Bank, and M&N Bank has no control over the charges as these charges are not levied by the bank.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "Whom should I approach for any other service related to M&N Bank Fixed Deposits",
+      "expected": "For any other service related to Fixed Deposits (Supersaver or Sweep-In Facility, addition of Nominee, Change in Maturity Instructions, Premature Encashment etc.) you may approach the nearest M&N Bank Branch with the Fixed Deposit advice and the request Form duly signed by all account holders.",
+      "predicted": "For any other service related to M&N Bank Fixed Deposits, you can write to us at www.M&N bank.com/services or contact your nearest PhoneBanking officer.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can I use my Medical benefit card outside India",
+      "expected": "No. Medical Benefit card is INR prepaid card which can be used only in India.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How do I obtain a duplicate NOC/Form 35 for a vehicle loan that has been closed and fully repaid",
+      "expected": "You may obtain a duplicate NOC/Form 35 for your vehicle by visiting any of our the Retail Loan Service Center and submit the following documents: Request form for Duplicate NOC/Form 35. Self attested copy of the Registration Certificate book, duly self attested by you, along with the Original Registration Certificate book for validation. Applicable fee of Rs. 500/- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer,i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided. Please note that the Loan Closure NOC/Form 35 will not be issued if any of the following conditions apply: Copy of RC Book is not submitted The same vehicle is encumbered to the Bank for any other loan facility Any notice from a competent authority restraining the Bank from release of hypothecation If there are any existing over dues, direct or otherwise, in your name",
+      "predicted": "You may obtain a duplicate NOC/Form 35 for your vehicle by visiting any of our the Retail Loan Service Center and submit the following documents: Request form for Duplicate NOC/Form 35. Self attested copy of the Registration Certificate book, duly self attested by you, along with the Original Registration Certificate book for validation. Applicable fee of Rs. 500/- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer,i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What happens if Premiums are not paid",
+      "expected": "This plan has a grace period of 30 days. You are expected to pay your annual premium through-out the policy term. Discontinuance before completion of 5 years from commencement of the policy: If you have not paid your premium by the expiry of the grace period, then you will have the following options: To revive the policy within a period of 2 years from the date of discontinuance, or To completely withdraw from the policy without any risk cover Your policy will be discontinued if: You do not exercise any of the above mentioned options; or You choose to completely withdraw from the policy without any risk cover Until the discontinuance of the policy, the risk cover will remain in-force and policy charges will continue to be deducted. Once the policy is discontinued, the risk cover will cease and the fund value (as on date of discontinuance) less the applicable Discontinuance Charge (Please see the Charges section for details of the Discontinuance Charges) will be moved to the 'Discontinued Policy Fund'. The minimum guaranteed interest rate applicable to the 'Discontinued Policy Fund' shall be 4% p.a. Such rate may be changed in the future if the IRDA revises the minimum rate for discontinued policies. The Discontinued Policy Fund (SFIN: ULIF05110/03/11DiscontdPF101) shall be a unit fund which will invest 100% of assets in Government Securities and/or Money Market Instruments. Any excess income earned in the discontinued fund over and above the minimum guaranteed interest rate shall also be apportioned to the 'Discontinued Policy Fund'. A Fund Management Charge of 0.50% p.a. will be levied for amounts in the 'Discontinued Policy Fund' If a discontinued policy is not revived, the proceeds will be paid out upon the completion of the lock-in period of five years. In the instances where the revival period is not completed at the end of the lock-in period, the policyholder can opt to receive the proceeds either upon the completion of the lock-in period or upon the completion of revival period. If, in such cases, the policyholder does not exercise any option, then the proceeds will be paid upon the completion of the lock-in period. After the payment of the discontinuance benefit, the policy shall terminate and no further benefits shall be payable under the policy. View more",
+      "predicted": "In case the Policyholder has not paid the premium by the expiry of the grace period, he/she will have the following options: to revive the Policy within a period of 2 years from the date of discontinuance, or to completely withdraw from the Policy without any risk cover, or to convert the policy into paid-up policy (available only on discontinuance after completion of the 5 policy years)",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "When does my policy acquire a Guaranteed Surrender Value (GSV)",
+      "expected": "It is advisable to continue your policy in order to enjoy full benefits of your policy. However, we understand that in certain circumstances you may want to surrender your policy.The policy will acquire a Guaranteed Surrender Value (GSV) provided 2 full years premium has been paid for premium payment term of less than 10 years and 3 full years premium have been paid for premium payment term of 10 years and above.The GSV shall be the aggregate of: Percentage of total premiums paid Percentage of accrued bonuses & accrued guaranteed additions Premiums will exclude any underwriting extra premiums and any taxes paid For details on GSV percentage (factors), please refer to the Product Brochure.",
+      "predicted": "The policy will acquire a Guaranteed Surrender Value (GSV) provided 2 full years premium has been paid. The GSV shall be aggregate of: Percentage of total premiums paid Percentage of accrued bonuses and accrued guaranteed additions Please refer the sales brochure or contact your Relationship manager to know the percentage factors for GSV",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the difference between Income and Income Plus Option",
+      "expected": "On the death of the Life Assured under Income Option, 10% of the death benefit is paid as lump sum upon death and the remaining 90% of the death benefit shall be paid as monthly income over next 15 years. On the death of Life Assured under Income Plus Option, 100% of the death benefit is payable as a lump sum benefit and a monthly income of 0.5% of sum assured is paid for 10 years. Also in this option, there is an option to choose Level Income or Increasing Income.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How soon can the card be used after it is purchased",
+      "expected": "Your card will be activated & loaded with required currency within 4 hours from the bank having received clear funds from you. The card will beready for use post activation.",
+      "predicted": "Your card will be activated & loaded with required currency within 4 hours from the bank having received clear funds from you. The card will be ready for use post activation.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What is the initial deposit amount required to open the Flexi Account",
+      "expected": "The initial deposit amount to open a Flexi Current account is Rs.75,000.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Is my overdraft limit fixed",
+      "expected": "This will completely depend on the market rate of the securities that you have pledged. Of course, we will value your portfolio periodically to determine your overdraft limit. You can increase your overdraft limit by pledging more shares/securities , or decrease it by withdrawing them that's completely up to you.",
+      "predicted": "No, your overdraft limit is not fixed. It's based on your account performance and salary credits.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is a settlement option",
+      "expected": "Under settlement option, you can also take your fund value at maturity in periodical installments.",
+      "predicted": "In our policy, the Settlement Option is when the Fund value is paid in periodical installments over a period which may extend up to 5 years. The value of installment payable on the date specified shall be subject to investment risk i.e. the NAV may go up or down depending upon the performance of the funds chosen by you.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What kind of benefits is my spouse and children entitled to",
+      "expected": "100% of the sum insured in case of accidental death, permanent disability, broken bones, burns,Hospital cash of the spouse & 10% of the sum insured in case of accidental death or permanent disability of the children.",
+      "predicted": "Under our banking's Family Plan, your spouse and children are eligible for a range of benefits, including life insurance coverage, accidental death and dismemberment (AD&D) protection, and other financial assistance in the event of an unexpected loss.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can a HUF/ Leased (Mhada/Cidco) property be insured",
+      "expected": "If you have financial interest in the property as owner or lessor then you can insure the property.",
+      "predicted": "Yes, a HUF (Hindu Succession Act) or leased property can be insured. However, the specific terms and conditions may vary depending on the type of policy and the insurance provider.\n\nIn general, most home insurance policies cover leased properties, including those owned by Mhada (Mumbai Housing Authority) or Cidco (City and Industrial Development Corporation).",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How do I top-up the value of my card when I am abroad",
+      "expected": "ForexPlus card can also be loaded on your behalf by someone authorized by you. Authorised person has to visit M&N Bank branch with the necessary documents and funds. Well take care of the rest.",
+      "predicted": "ISIC M&N Bank ForexPlus Card can also be loaded on your behalf by someone authorized (blood relatives only) by you. Authorised person has to visit M&N Bank branch with the necessary documents and funds. Well take care of the rest. You can reload your ISIC ForexPlus Card multiple times in future till the card gets expired: You can authorize the Bank to debit your M&N Bank account by giving an M&N Bank cheque or make payment using a local cheque drawn on M&N Bank and reload the card for desired value. Reload requests can be made by any authorized person (blood relatives only) on your behalf. The person authorised by you for the same will need to come to bank branch and will need to provide the necessary documents to the bank to carry out reload. On receipt of the application form and the funds, your card will get reloaded for the desired amount. M&N Bank customers can also place a request to reload the card through M&N Bank PhoneBanking or M&N Bank NetBanking.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the limited mode of premium pay",
+      "expected": "A limited premium payment policy requires you to pay premiums for a term lesser than your policy term.",
+      "predicted": "With M&N Life ClassicAssure Plus, you can pay premiums for 7 years for policy terms of 10 and 15 years or pay premiums for 10 years for policy term of 15 and 20 years.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Where do I get this card",
+      "expected": "Hajj Umrah ForexPlus Card is available over the counter at M&N Bank branches. You simply need to walk into our branch with required documentation and walk out with the card. The card will get activated within 5 to 6 hours from realization of funds.",
+      "predicted": "ISIC ForexPlus Card is available over the counter at M&N Bank branches. You simply need to walk into our branch with required documentation and walk out with ISIC ForexPlus Card. The card will get activated within 4 hours from realization of funds. Also you can apply by visiting www.M&N bank.com -> Apply Online section or www.M&N bank.com/personal/products/cards/prepaid-cards -> ISIC Student ID ForexPlus Card -> Buy Forex Card",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "How does it work",
+      "expected": "The Super Saver Account lets you to withdraw up to 90% of your deposit, just like the liquidity of a Savings Account. Theres no need to break your Fixed Deposit for short term requirements. You can withdraw cash through an ATM or by cheque or teller transactions.",
+      "predicted": "Verified by Visa/ MasterCard SecureCode provides you with a Password/ SecureCode that protects your online transactions, just like the way you use your PIN at an ATM. All you need to do is follow the simple steps: Select the goods and services you want from the Verified by Visa (VBV) / SecureCode merchant site and proceed to the payment page. Enter your M&N Bank Debit/ Credit Card number and the online store will connect with M&N Bank to check whether your Card is secured with Verified by Visa/ MasterCard SecureCode. M&N Bank initiates a Verified by Visa/ MasterCard SecureCode pop-up window on your computer screen that includes your Personal Assurance Message (PAM) or Personal Greeting message. Look for and confirm your PAM/ Personal Greeting then enter your password. M&N Bank will then confirm your identity to the merchant. Your payment is authorized and your order is placed.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What do you mean by exclusions under the Accidental Protection Plan - Hospital Cash",
+      "expected": "Exclusions are situations or conditions where M&N ERGO is not liable to pay the benefits to the insured person in the event of an accident.",
+      "predicted": "Could you tell me more about what you're looking for? Are you referring to a specific exclusion or condition that is not covered under the policy, such as pre-existing medical conditions or certain types of accidents?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can I repay the loan earlier for Professionals Loan",
+      "expected": "Yes, you can choose to repay your loan anytime after 06 months of taking the loan and after you have paid 06 EMIs along with applicable foreclosure charges. Please note Loan can be foreclosed by issuing a Demand draft / cheque at the branch; kindly note that cash beyond Rs. 49000/- will not be accepted for loan repayment against full closure/matured loan closure payment. Any payment above Rs. 49000/- should be made vide Account payee Cheque / Pay order /Demand draft.",
+      "predicted": "Yes, you can choose to repay your loan anytime after 12 months of taking the loan and after you have paid 12 EMIs along with applicable foreclosure charges.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the Maturity Benefit under the plan",
+      "expected": "On maturity, the policy holder will receive the sum assured plus accrued bonuses on full payment of premiums due throughout the premium paying term.",
+      "predicted": "Fund Value on the prevailing on the date of Maturity will be paid to the policyholder.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How do I know if I am eligible for the education loan for Indian education",
+      "expected": "If you are an Indian resident/citizen, have got admission in a graduate/ post graduate course in a college/institute/university which is recognized by the UGC/AICTE/AIBMS or ICMR, then you are eligible for the M&N Bank Student Loan for Indian Education.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Do I need to have M&N Bank account to purchase the Regalia ForexPlus Card",
+      "expected": "No, you don't need to have an account with M&N Bank to purchase Regalia ForexPlus Card. You can just walk into any of our M&N bank branches in India and purchase it over the counter.",
+      "predicted": "No, you don't need to have an account with M&N Bank to purchase the Regalia ForexPlus Card. You can just walk into any of our M&N bank branches in India and purchase it over the counter on submission of relevant KYC documents.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "Who can trade in derivatives with an M&N Securities Trading Account",
+      "expected": "Trading in Derivatives will be available only to those customers who have submitted their financial documents or Income proof and signed the necessary agreements.",
+      "predicted": "Trading in Derivatives will be available only to those customers who have submitted their financial documents or income proof and signed the necessary agreements. For more information click here. For more information visit www.M&N sec.com",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "Who do I contact to submit a claim under Health Suraksha policy in case of hospitalisation",
+      "expected": "For hospitalization /claim, customer can contact via Call: 1800 200 1999 / 1860 2000 700 Fax: 1860 2000 600 Email: [email protected] Through IPO: Login to Insurance Portfolio Organizer (IPO) from M&N Ergo website home page",
+      "predicted": "You should immediately intimate us on our helpline number regarding a claim under the Health Suraksha policy in case of hospitalization.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Will I be able to access both my accounts at all establishments/ATMs through a single Debit Card",
+      "expected": "Your existing Debit Card will be linked to both accounts. Any additional card will be charged at the regular rates. You will be able to select either account to transact at M&N Bank ATMs. At other VISA/PLUS ATMs (Other Banks ATMs) you may or may not have access to both accounts. When such an option is unavailable, you can only access your salary account. If you are a VISA Electron Debit Cardholder you can access only your Salary Account at merchant locations for transactions and not your Reimbursement Account. If you are a Maestro Debit Cardholder may be able to select between your Salary Account and Reimbursement Account at merchant locations.",
+      "predicted": "Both account holders can get an ATM / Debit Card, but a maximum of two cards are given for each Super Saver Account. This means you'll need to use separate ATMs or Debit Cards to access your individual accounts if you have more than one joint account holder.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How do I make my premium payments",
+      "expected": "This is an annual policy, where the premium can be made via Credit Card or Cheque",
+      "predicted": "This is an annual policy, where the premium can be made online via Credit Card/Netbanking or Cheque.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "Can I open a joint account with a non-senior citizen for 5 Year Tax Saving Fixed Deposit",
+      "expected": "Yes you can open deposit with non senior citizen.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "how much is interest for FD",
+      "expected": "7%",
+      "predicted": "The interest rate for Fixed Deposit (FD) is 7%.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What is the minimum balance required for an Ultima Current account",
+      "expected": "You need to maintain an Average Quarterly Balance (AQB) of Rs. 20,00,000 in your Ultima Current account.",
+      "predicted": "I don't have information on that",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What do I do if I am not able to avail merchant offers",
+      "expected": "Do not worry if you are not able to avail the offers at your disposal. Any issue related merchant or code not working or bar code not getting read, just write to [email protected]",
+      "predicted": "Could you tell me more about what you're looking for? Are you referring to a specific account type or product that has merchant offers, and are you unable to take advantage of them?",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can I load multiple currencies on one Regalia ForexPlus Card",
+      "expected": "No. Your Regalia ForexPlus Card can only be loaded in USD currency. As a feature of the card, no cross currency conversion charges will be levied for using the card in any currency other than USD. Your transaction value will get converted in USD currency @ rate applied by MasterCard. No additional cost will be levied.",
+      "predicted": "I don't have information on that",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can I take a loan on these bonds",
+      "expected": "No, you cannot take a loan on these bonds.",
+      "predicted": "Policy loan is not offered in this plan.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Do I need to provide collateral to avail of an education loan (Education Loans For Indian Education)",
+      "expected": "For loans up to Rs. 4 lakh - No collateral or Third Party Guarantee is required For loans from Rs. 4 lakh to Rs. 7.5 lakh No collateral required, however Third Party Guarantee is required For loans above Rs. 7.5 lakh collateral is required. You can choose from any of the following acceptable collaterals: Residential Property M&N Bank Fixed Deposit LIC/NSC/KVP View more",
+      "predicted": "For loans up to Rs. 4 lakh, No collateral or Third Party Guarantee is required.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "What does 3D stand for in M&N Life Click 2 Protect 3D Plus",
+      "expected": "3D stands for Death, Disease & Disability. M&N Life Click 2 Protect 3D Plus is a comprehensive term plan that provides protection against these 3D's.",
+      "predicted": "The context doesn't explicitly state what \"3D\" stands for, but it does mention that the plan options provide Waiver of Premium benefit if the Life Assured is diagnosed with any of the 34 covered Critical Illnesses.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "How do I claim Personal Accident Death Cover for Gold Debit Card",
+      "expected": "Nominees and legal heirs should visit their nearest M&N Bank branch. We will help with the documents required, and, when these are complete, we will liaise with the relevant insurance company to resolve the claim. This is a special service for our EasyShop Gold Debit Card holders. Remember to provide us with all documents within one month of death.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Can I use the card to pay in any currency",
+      "expected": "Yes, the card is accepted for payment in any currency apart from those loaded on the card. No cross currency conversion charges will be applicable while processing such transactions. - Exchange rate used will be the MasterCard wholesale exchange rate prevailing at the time of the transaction. - Exchange rate used will be the Midrate prevailing at the time of the transaction.",
+      "predicted": "Yes, the card is accepted for payment in any currency apart from those loaded on the card. Please note that cross currency conversion charges will be applicable while processing such transactions.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "Do I need to lodge an FIR in case of an accident",
+      "expected": "You will need to lodge an FIR in the following circumstances: Any third party personal injury or death due to an accident involving your vehicle Any personal injury or death of a paid driver Any personal injury or death of an owner-driver Any personal injury or death of a passenger Any third party property damage Theft of the entire vehicle Theft of accessories, stereo-recorders, other vehicle parts",
+      "predicted": "No, according to the context provided, a police FIR is not mentioned as being necessary for accidents. However, it's worth noting that the context only mentions specific types of claims (Burglary, Theft, Malicious Damage, Riot and Strike, Terrorism) where an FIR is mandatory. It does not explicitly state what happens in case of an accident claim under Home Insurance policy.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    },
+    {
+      "query": "Do I need to have an M&N Bank account to purchase M&N Bank Multi Currency ForexPlus Platinum Card",
+      "expected": "No, you don't need to have an account with M&N Bank to purchase the M&N Bank Multi Currency ForexPlus Platinum Card. You can just walk into any of our M&N bank branches in India and purchase it over the counter.",
+      "predicted": "No, you don't need to have an account with M&N Bank to purchase the Multicurrency card. You can just walk into any of our M&N bank branches in India and purchase it over the counter.",
+      "retrieval_success": false,
+      "correct": true,
+      "hallucinated": false
+    },
+    {
+      "query": "What are the documents required",
+      "expected": "Please refer Policy wordings for exhaustive list of claims documents required. We will not ask for anything over and above the list given in Policy.",
+      "predicted": "For Salaried Individuals, the documents required to avail of the Loan Against Property are:\n\n1. Proof of Residence - Any one of Ration Card / Telephone Bill / Electricity Bill / Voters Card.\n2. Proof of Identity - Any one of Voters Card / Drivers License / Employers Card.\n3. Latest Bank Statement / Passbook (where salary / income is credited for past 6 months).\n4. Latest 6 Months Salary Slip with all deductions and last 2 years Form 16.\n5. Copies of all Property Documents.\n\nFor Self-Employed Individuals, the documents required are:\n\n1. Certified Financial Statement for the last 2 years.\n2. Proof of Residence - Any one of Ration Card / Telephone Bill / Electricity Bill / Voters Card.\n3. Proof of Identity - Any one of Voters Card / Drivers License / Employers Card.\n4. Latest Bank Statement / Passbook (where salary / income is credited for past 6 months).\n5. Copies of all Property Documents.\n\nPlease note that these documents are required to open a Salary Account as well, but the specific requirements may vary depending on the type of account and individual circumstances.",
+      "retrieval_success": false,
+      "correct": false,
+      "hallucinated": true
+    }
+  ]
+}

test_eval_v7.json

@@ -0,0 +1,431 @@
+{
+  "metrics": {
+    "split": "test",
+    "accuracy": 0.7,
+    "hallucination_rate": 0.05,
+    "retrieval_failure_rate": 0.35,
+    "samples": 20
+  },
+  "detailed_results": [
+    {
+      "query": "How do I become eligible for an M&N Bank Institutional Current Account",
+      "expected": "To become eligible for an M&N Bank Institutional Current Account you belong to one of the eligibility criteria mentioned above.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": true,
+      "correct": false,
+      "score": 0.0,
+      "reason": "Model returned a fallback response",
+      "is_fallback": true,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Model does not know the answer"
+      },
+      "confidence_scores": [
+        0.7672,
+        0.7565,
+        0.7177
+      ],
+      "avg_confidence": 0.7471
+    },
+    {
+      "query": "Can I issue cheques to load the Multicurrency Platinum ForexPlus Chip card",
+      "expected": "Customer can load Multicurrency card by issuance of cheque against the amount to be loaded. However, cards will be loaded post realisation of the cheque deposited by the customer. The exchange rate applicable will be for the day on which funds are realized and card is activated.",
+      "predicted": "Yes, you can issue cheques to load the Multicurrency Platinum ForexPlus Chip card.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "Most key information is present in the predicted answer, but it lacks specificity about when the card is loaded post cheque realization.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7942,
+        0.7877,
+        0.77
+      ],
+      "avg_confidence": 0.784
+    },
+    {
+      "query": "What do you mean by exclusions under the Accidental Protection Plan - Hospital Cash",
+      "expected": "Exclusions are situations or conditions where M&N ERGO is not liable to pay the benefits to the insured person in the event of an accident.",
+      "predicted": "Exclusions under the Accidental Protection Plan - Hospital Cash policy are situations or conditions where M&N ERGO is not liable to pay the benefits to the insured person in the event of an accident.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 1.0,
+      "reason": "The predicted answer covers all key points in the expected answer with minor wording differences.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7486,
+        0.7469,
+        0.7451
+      ],
+      "avg_confidence": 0.7469
+    },
+    {
+      "query": "Can I get an overdraft facility on my fixed deposit funds",
+      "expected": "With our Super Saver Account, you can enjoy an overdraft facility of up to 90% of your Fixed Deposit funds without breaking the Fixed Deposit or losing interest on it. However, a minimum tenure of 6 months is required to open a Super Saver Fixed Deposit/ avail of the Super Saver facility on your existing deposit.",
+      "predicted": "Yes, with a Sweep-in Fixed Deposit linked to your Savings Account, you can avail of an Overdraft facility up to 75% of the principal amount of the Fixed Deposit. The rate applicable on this Overdraft facility will be 2% per annum over and above the Fixed Deposit Rate.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "The predicted answer covers most of the key points in the expected answer, but lacks specificity on the interest rate and minimum tenure requirements.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7984,
+        0.7317,
+        0.7026
+      ],
+      "avg_confidence": 0.7442
+    },
+    {
+      "query": "Do I need to undergo medical test for buying this policy",
+      "expected": "You are not required to undergo any medical tests up to the age of 55 years, except if you have declared any pre-existing diseases or ailments at the time of applying for the policy. In such cases and for applicants above age 55 years, one has to undergo the specified medical tests.",
+      "predicted": "No pre-policy medical check up is required for individuals up to the age of 45 years.",
+      "retrieval_success": false,
+      "correct": true,
+      "score": 0.8,
+      "reason": "The predicted answer is mostly accurate but misses the age limit of 55 years for medical tests.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7628,
+        0.7548,
+        0.722
+      ],
+      "avg_confidence": 0.7465
+    },
+    {
+      "query": "What is M&N ERGO General Insurance role in a third party injury/property damage suit",
+      "expected": "We will handle the legal suit from the moment you receive a notification till the case is resolved. We will either settle the case out of court or actively defend the matter at the Motor Accident Claims Tribunal. Your co-operation will be essential in gathering the necessary documents and preparing a defence viz. vehicle registration papers, driving licence of the person driving the vehicle, police FIR and panchnama, vakalatnama signed by you, your written statement as to the facts of the accident. We will also take care of legal liability and costs awarded by the tribunal. However, we will not be able to help you in the following situations: Where your insurance policy is not valid, either because it has expired or because the cheque for your premium has been dishonoured. The person driving the vehicle at the time of the accident was under the influence of drugs or alcohol. The person driving the vehicle at the time of the accident did not hold a valid driving licence to drive the said vehicle. View more",
+      "predicted": "According to my knowledge, as per the context provided, M&N ERGO General Insurance's role in a third-party injury or property damage suit would be to reimburse the insured person for the amount payable to them by the third party, or pay the garage directly if it's a cashless settlement.",
+      "retrieval_success": false,
+      "correct": true,
+      "score": 0.8,
+      "reason": "The predicted answer covers most of the key points in the expected answer, including M&N ERGO General Insurance's role and responsibilities, but misses some details such as the specific situations where they will not help.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7407,
+        0.7005,
+        0.6929
+      ],
+      "avg_confidence": 0.7114
+    },
+    {
+      "query": "What is the difference between 3D Life and 3D Life Long Protection options",
+      "expected": "In addition to the benefits under 3D Life Option, Life Assured is covered for whole life under 3D Life Long Protection option. Premiums under this option have to be paid only till an age of 65.",
+      "predicted": "The context does not explicitly state the differences between 3D Life and 3D Life Long Protection options, but it mentions that both are part of the plan options available under this plan. However, I can infer from the information provided earlier that the main difference is that Life Long Protection option provides coverage for whole life, whereas no specific details about 3D Life option are mentioned in the context.\n\nIf you could tell me more about what you're looking for regarding these two options (e.g., benefits, exclusions, premium requirements), I'd be happy to try and help further.",
+      "retrieval_success": true,
+      "correct": false,
+      "score": 0.0,
+      "reason": "Predicted does not match with expected",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": true,
+        "confidence": "low",
+        "reason": "The response contains unsupported claims as it provides information not explicitly stated in the context."
+      },
+      "confidence_scores": [
+        0.7553,
+        0.7539,
+        0.7239
+      ],
+      "avg_confidence": 0.7444
+    },
+    {
+      "query": "Which courses can I take an education loan for (Education Loans For Foreign Education)",
+      "expected": "You can take a Education Loan for foreign education on the following courses: Management Courses (Full-Time) Engineering Courses Medicine - Graduation + Post Graduation Masters in Computer Application ( MCA / MCM ) Architecture Hotel and Hospitality Agriculture Pure Science View more",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": true,
+      "correct": false,
+      "score": 0.0,
+      "reason": "Model returned a fallback response",
+      "is_fallback": true,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Model does not know the answer"
+      },
+      "confidence_scores": [
+        0.8209,
+        0.7939,
+        0.782
+      ],
+      "avg_confidence": 0.7989
+    },
+    {
+      "query": "What are the currencies available for M&N Bank ForexPlus Platinum card",
+      "expected": "The M&N Bank ForexPlus Platinum card is currently available in 5 currencies, i.e. USD, Euro, GBP, HKD and THB.",
+      "predicted": "You can carry following currencies on your M&N Bank Multi Currency ForexPlus Platinum Card - Australian Dollar, United Arab Emirates Dirham, Canadian Dollar, Euro, Japanese Yen, Singapore Dollar, Sterling Pound, Swiss Franc, Swedish Krona and US Dollar.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "Most currencies mentioned in the predicted answer match the expected answer, but some currencies (Australian Dollar, United Arab Emirates Dirham, Canadian Dollar, Japanese Yen, Singapore Dollar, Sterling Pound, Swiss Franc, Swedish Krona) are not available for M&N Bank ForexPlus Platinum card.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.8525,
+        0.8109,
+        0.7936
+      ],
+      "avg_confidence": 0.819
+    },
+    {
+      "query": "What is Extra Life Option in the policy",
+      "expected": "You can opt for Extra life Option under Classic Benefit under the plan. In case of death due to accident during the term of the policy, an additional Sum Assured will be payable to the Nominee in addition to the Death Benefit under the Plan. Please refer to product brochure for further details and/or consult Relationship Manager.",
+      "predicted": "In the policy, Extra Life Option provides an additional benefit of Extra Life Sum Assured upon death due to Accident. This means that if the policyholder dies due to an accident, they will receive not only the base sum assured but also an extra amount (up to the maximum allowed) as a separate benefit.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "Most key information is present in the predicted answer, but some minor details (e.g., 'Classic Benefit' and 'Refer to product brochure') are missing or not explicitly stated.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7913,
+        0.7334,
+        0.7293
+      ],
+      "avg_confidence": 0.7513
+    },
+    {
+      "query": "How do I obtain a NOC from the bank if I have lost my Registration Certificate book for a vehicle under finance with M&N Bank",
+      "expected": "You may obtain the NOC for a duplicate Registration Certificate book for your vehicle loan, by visiting any of our Retail Loan Service Center and submitting the following documents: Request form for applying for the required NOC. Self attested copy of the Registration Certificate book, duly self attested by you for validation. Copy of FIR or certificate from police or Acknowledgement from Police on written complaint with General Diary No. Applicable fee of Rs. 500//- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer,i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided. View more",
+      "predicted": "You may obtain the NOC for a duplicate Registration Certificate book for your vehicle loan, by visiting any of our Retail Loan Service Center and submitting the following documents: Request form for applying for the required NOC. Self attested copy of the lost Registration Certificate book, duly self attested by you, along with the Original Registration Certificate book for validation. Applicable fee of Rs. 500/- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer, i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.9,
+      "reason": "Most key information is present in both answers, with minor differences in formatting and wording.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.8565,
+        0.8536,
+        0.7733
+      ],
+      "avg_confidence": 0.8278
+    },
+    {
+      "query": "Can I avail an Overdraft against my Property",
+      "expected": "A Loan Against Property can be availed as a Term Loan with EMI repayments or as a Drop line Overdraft Facility.",
+      "predicted": "No, you cannot avail an Overdraft against your Property. The context only mentions Loan Against Securities and Loan Against Property as two separate account options. There is no information about an Overdraft being available against a property.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "The predicted answer covers most of the key points in the expected answer, including Loan Against Property as a term loan or drop line overdraft facility.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7226,
+        0.7002,
+        0.6911
+      ],
+      "avg_confidence": 0.7046
+    },
+    {
+      "query": "Where do I get this card",
+      "expected": "ForexPlus card is available over the counter at M&N Bank branches. You simply need to walk into our branch with required documentation and walk out with ForexPlus card. The card will get activated within 4 hours from realization of funds. Alternatively, the customer can apply for card through NetBanking login also.In case the customer applies the card through NetBanking the card will be delivered to the customer within 2 working days. This facility is available at select locations only.",
+      "predicted": "You can obtain the ForexPlus Chip card over the counter at M&N Bank branches. Simply walk into a branch with required documentation and you'll be able to take the card away with you. The activation process typically takes 4 hours from the realization of funds.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "Most key information is present in the predicted answer, but some minor details and formatting differences are missing or altered.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.705,
+        0.7037,
+        0.7011
+      ],
+      "avg_confidence": 0.7033
+    },
+    {
+      "query": "Will I get an online confirmation of my transactions",
+      "expected": "An order once placed will be confirmed immediately. The confirmation of units bought or sold will be done by the AMC / Registrar only on the next working day.",
+      "predicted": "Yes, when you use Verified by Visa/ MasterCard SecureCode, you will receive a pop-up window on your computer screen with a Personal Assurance Message (PAM) or Personal Greeting message from M&N Bank, confirming the transaction.",
+      "retrieval_success": false,
+      "correct": true,
+      "score": 0.8,
+      "reason": "Most key information is present in both answers, but the predicted answer lacks specific details about AMC/Registrar confirmation.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7425,
+        0.7208,
+        0.6853
+      ],
+      "avg_confidence": 0.7162
+    },
+    {
+      "query": "What are the documents required",
+      "expected": "Please refer Policy wordings for exhaustive list of claims documents required. We will not ask for anything over and above the list given in Policy.",
+      "predicted": "For opening a Current Account of a company, the following documents are required:\n\n1. Memorandum of Association (MOA)\n2. Articles of Association (AOA)\n3. Certificate of Incorporation\n4. Latest list of directors duly signed by any Director/Company Secretary /Authorised Signatory\n5. Board Resolution (BR) duly signed by the directors of the company.\n\nFor opening a Current Account of a partnership firm, the following documents are required:\n\n1. Latest Partnership Deed\n2. Documents establishing proof of existence\n3. Registration Certificate issued by Registrar of Firms\n4. Valid Business License or Certificate of Registration issued by State/Central Government authority\n5. PAN Card in the name of the firm\n6. Self-signed cheque issued from the firms account from a PSU/private sector/foreign bank along with a bank statement not older than four months\n7. Documents for establishing proof of address (such as landline telephone bill/electricity bill)\n8. Property Ownership Deed i.e. title deeds of the property in the name of the firm duly stamped and registered\n9. Latest property tax or water tax paid receipt/bill raised in the name of the firm\n10. TAN Allotment Letter in the name of the firm\n11. Existing bank account statement or passbook of a PSU/private sector/foreign bank\n\nFor opening a Current Account of a sole proprietorship firm, the following documents are required:\n\n1. Proof of existence in the name of firm\n2. Proof of address in the name of firm\n3. KYC of the proprietor\n4. Any two of the below listed documents for establishing proof of existence (such as registration certificate/license issued by Municipal authorities, etc.)",
+      "retrieval_success": false,
+      "correct": false,
+      "score": 0.0,
+      "reason": "Predicted does not match with expected",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "The response contains a detailed list of required documents for opening Current Accounts in different types of firms, which is not supported by the context."
+      },
+      "confidence_scores": [
+        0.7192,
+        0.7079,
+        0.7067
+      ],
+      "avg_confidence": 0.7113
+    },
+    {
+      "query": "How can I make a request to foreclose my Loans Against Rental Receivables account",
+      "expected": "Kindly submit a written request to your nearest Retail Loan Service Centre to close your account. Our customer service desk will provide you with the necessary assistance.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "score": 0.0,
+      "reason": "Model returned a fallback response",
+      "is_fallback": true,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Model does not know the answer"
+      },
+      "confidence_scores": [
+        0.7408,
+        0.7237,
+        0.6976
+      ],
+      "avg_confidence": 0.7207
+    },
+    {
+      "query": "Will I be able to access both my accounts at all establishments/ATMs through a single Debit Card",
+      "expected": "Your existing Debit Card will be linked to both accounts. Any additional card will be charged at the regular rates. You will be able to select either account to transact at M&N Bank ATMs. At other VISA/PLUS ATMs (Other Banks ATMs) you may or may not have access to both accounts. When such an option is unavailable, you can only access your salary account. If you are a VISA Electron Debit Cardholder you can access only your Salary Account at merchant locations for transactions and not your Reimbursement Account. If you are a Maestro Debit Cardholder may be able to select between your Salary Account and Reimbursement Account at merchant locations.",
+      "predicted": "Yes, as a joint Super Saver Account holder, you can get an ATM / Debit Card that allows access to both accounts at most establishments/ATMs.",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "The predicted answer covers most of the key points in the expected answer, but lacks specificity about VISA/PLUS ATMs and merchant locations.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.6918,
+        0.6886,
+        0.6799
+      ],
+      "avg_confidence": 0.6868
+    },
+    {
+      "query": "How do I know my application has been accepted",
+      "expected": "You can check your account opening status on www.M&N sec.com on the footer page of the homepage section page under the Application status.",
+      "predicted": "You will be informed once your application is accepted and all the required accounts are set up. In case your application is not processed because of lack of some details, our representative will contact you by mail.",
+      "retrieval_success": false,
+      "correct": true,
+      "score": 0.8,
+      "reason": "The predicted answer covers most of the key points in the expected answer but lacks specific details about checking account status online.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7143,
+        0.68,
+        0.6228
+      ],
+      "avg_confidence": 0.6724
+    },
+    {
+      "query": "How do Recurring Deposits work",
+      "expected": "A monthly deposit of a pre-fixed amount is made in the Recurring Deposit account. On maturity, you get back the amount you deposited, along with interest calculated at the rate applicable at the time of opening your account.",
+      "predicted": "Recurring Deposits can now be booked through NetBanking. Following are the steps for booking your Recurring Deposit online: Access your NetBanking account with your Customer ID and IPIN (NetBanking Password) Select Open Recurring Deposit option under the Recurring Deposit menu from the Menu bar located on the left hand side of the web page Select the desired values from the Drop-down lists Once complete, click on Continue and Confirm the details entered",
+      "retrieval_success": true,
+      "correct": true,
+      "score": 0.8,
+      "reason": "The predicted answer covers most of the key points in the expected answer, but lacks specific details about interest calculation and maturity amount.",
+      "is_fallback": false,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Response is correct"
+      },
+      "confidence_scores": [
+        0.7399,
+        0.7276,
+        0.7272
+      ],
+      "avg_confidence": 0.7316
+    },
+    {
+      "query": "How much am I charged for this service",
+      "expected": "A quarterly maintenance charge is applicable to hold an Investment Services Account. We request you to contact your RM / Personal Banker or nearest M&N Bank Branch for details.",
+      "predicted": "I don't have information on that.",
+      "retrieval_success": false,
+      "correct": false,
+      "score": 0.0,
+      "reason": "Model returned a fallback response",
+      "is_fallback": true,
+      "hallucination_detail": {
+        "hallucinated": false,
+        "confidence": "high",
+        "reason": "Model does not know the answer"
+      },
+      "confidence_scores": [
+        0.6953,
+        0.6919,
+        0.6699
+      ],
+      "avg_confidence": 0.6857
+    }
+  ]
+}