test_eval_v6.json

{
  "metrics": {
    "split": "test",
    "accuracy": 0.7,
    "hallucination_rate": 0.05,
    "retrieval_failure_rate": 0.35,
    "samples": 20
  },
  "detailed_results": [
    {
      "query": "How do I become eligible for an M&N Bank Institutional Current Account",
      "expected": "To become eligible for an M&N Bank Institutional Current Account you belong to one of the eligibility criteria mentioned above.",
      "predicted": "I don't have information on that.",
      "retrieval_success": true,
      "correct": false,
      "score": 0.0,
      "reason": "Model returned a fallback response",
      "is_fallback": true,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Model does not know the answer"
      },
      "confidence_scores": [
        0.7672,
        0.7565,
        0.7177
      ],
      "avg_confidence": 0.7471
    },
    {
      "query": "Can I issue cheques to load the Multicurrency Platinum ForexPlus Chip card",
      "expected": "Customer can load Multicurrency card by issuance of cheque against the amount to be loaded. However, cards will be loaded post realisation of the cheque deposited by the customer. The exchange rate applicable will be for the day on which funds are realized and card is activated.",
      "predicted": "Yes, you can issue cheques to load the Multicurrency Platinum ForexPlus Chip card.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "Most key information is present in the predicted answer, but it lacks specificity about when the card is loaded post cheque realization.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7942,
        0.7877,
        0.77
      ],
      "avg_confidence": 0.784
    },
    {
      "query": "What do you mean by exclusions under the Accidental Protection Plan - Hospital Cash",
      "expected": "Exclusions are situations or conditions where M&N ERGO is not liable to pay the benefits to the insured person in the event of an accident.",
      "predicted": "Exclusions under the Accidental Protection Plan - Hospital Cash policy are situations or conditions where M&N ERGO is not liable to pay the benefits to the insured person in the event of an accident.",
      "retrieval_success": true,
      "correct": true,
      "score": 1.0,
      "reason": "The predicted answer covers all key points in the expected answer with minor wording differences.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7486,
        0.7469,
        0.7451
      ],
      "avg_confidence": 0.7469
    },
    {
      "query": "Can I get an overdraft facility on my fixed deposit funds",
      "expected": "With our Super Saver Account, you can enjoy an overdraft facility of up to 90% of your Fixed Deposit funds without breaking the Fixed Deposit or losing interest on it. However, a minimum tenure of 6 months is required to open a Super Saver Fixed Deposit/ avail of the Super Saver facility on your existing deposit.",
      "predicted": "Yes, with a Sweep-in Fixed Deposit linked to your Savings Account, you can avail of an Overdraft facility up to 75% of the principal amount of the Fixed Deposit. The rate applicable on this Overdraft facility will be 2% per annum over and above the Fixed Deposit Rate.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "The predicted answer covers most of the key points in the expected answer, but lacks specificity on the interest rate and minimum tenure requirements.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7984,
        0.7317,
        0.7026
      ],
      "avg_confidence": 0.7442
    },
    {
      "query": "Do I need to undergo medical test for buying this policy",
      "expected": "You are not required to undergo any medical tests up to the age of 55 years, except if you have declared any pre-existing diseases or ailments at the time of applying for the policy. In such cases and for applicants above age 55 years, one has to undergo the specified medical tests.",
      "predicted": "No pre-policy medical check up is required for individuals up to the age of 45 years.",
      "retrieval_success": false,
      "correct": true,
      "score": 0.8,
      "reason": "The predicted answer is mostly accurate but misses the age limit of 55 years for medical tests.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7628,
        0.7548,
        0.722
      ],
      "avg_confidence": 0.7465
    },
    {
      "query": "What is M&N ERGO General Insurance role in a third party injury/property damage suit",
      "expected": "We will handle the legal suit from the moment you receive a notification till the case is resolved. We will either settle the case out of court or actively defend the matter at the Motor Accident Claims Tribunal. Your co-operation will be essential in gathering the necessary documents and preparing a defence viz. vehicle registration papers, driving licence of the person driving the vehicle, police FIR and panchnama, vakalatnama signed by you, your written statement as to the facts of the accident. We will also take care of legal liability and costs awarded by the tribunal. However, we will not be able to help you in the following situations: Where your insurance policy is not valid, either because it has expired or because the cheque for your premium has been dishonoured. The person driving the vehicle at the time of the accident was under the influence of drugs or alcohol. The person driving the vehicle at the time of the accident did not hold a valid driving licence to drive the said vehicle. View more",
      "predicted": "According to my knowledge, as per the context provided, M&N ERGO General Insurance's role in a third-party injury or property damage suit would be to reimburse the insured person for the amount payable to them by the third party, or pay the garage directly if it's a cashless settlement.",
      "retrieval_success": false,
      "correct": true,
      "score": 0.8,
      "reason": "The predicted answer covers most of the key points in the expected answer, including M&N ERGO General Insurance's role and responsibilities, but misses some details such as the specific situations where they will not help.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7407,
        0.7005,
        0.6929
      ],
      "avg_confidence": 0.7114
    },
    {
      "query": "What is the difference between 3D Life and 3D Life Long Protection options",
      "expected": "In addition to the benefits under 3D Life Option, Life Assured is covered for whole life under 3D Life Long Protection option. Premiums under this option have to be paid only till an age of 65.",
      "predicted": "The context does not explicitly state the differences between 3D Life and 3D Life Long Protection options, but it mentions that both are part of the plan options available under this plan. However, I can infer from the information provided earlier that the main difference is that Life Long Protection option provides coverage for whole life, whereas no specific details about 3D Life option are mentioned in the context.\n\nIf you could tell me more about what you're looking for regarding these two options (e.g., benefits, exclusions, premium requirements), I'd be happy to try and help further.",
      "retrieval_success": true,
      "correct": false,
      "score": 0.0,
      "reason": "Predicted does not match with expected",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": true,
        "confidence": "low",
        "reason": "The response contains unsupported claims as it provides information not explicitly stated in the context."
      },
      "confidence_scores": [
        0.7553,
        0.7539,
        0.7239
      ],
      "avg_confidence": 0.7444
    },
    {
      "query": "Which courses can I take an education loan for (Education Loans For Foreign Education)",
      "expected": "You can take a Education Loan for foreign education on the following courses: Management Courses (Full-Time) Engineering Courses Medicine - Graduation + Post Graduation Masters in Computer Application ( MCA / MCM ) Architecture Hotel and Hospitality Agriculture Pure Science View more",
      "predicted": "I don't have information on that.",
      "retrieval_success": true,
      "correct": false,
      "score": 0.0,
      "reason": "Model returned a fallback response",
      "is_fallback": true,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Model does not know the answer"
      },
      "confidence_scores": [
        0.8209,
        0.7939,
        0.782
      ],
      "avg_confidence": 0.7989
    },
    {
      "query": "What are the currencies available for M&N Bank ForexPlus Platinum card",
      "expected": "The M&N Bank ForexPlus Platinum card is currently available in 5 currencies, i.e. USD, Euro, GBP, HKD and THB.",
      "predicted": "You can carry following currencies on your M&N Bank Multi Currency ForexPlus Platinum Card - Australian Dollar, United Arab Emirates Dirham, Canadian Dollar, Euro, Japanese Yen, Singapore Dollar, Sterling Pound, Swiss Franc, Swedish Krona and US Dollar.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "Most currencies mentioned in the predicted answer match the expected answer, but some currencies (Australian Dollar, United Arab Emirates Dirham, Canadian Dollar, Japanese Yen, Singapore Dollar, Sterling Pound, Swiss Franc, Swedish Krona) are not available for M&N Bank ForexPlus Platinum card.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.8525,
        0.8109,
        0.7936
      ],
      "avg_confidence": 0.819
    },
    {
      "query": "What is Extra Life Option in the policy",
      "expected": "You can opt for Extra life Option under Classic Benefit under the plan. In case of death due to accident during the term of the policy, an additional Sum Assured will be payable to the Nominee in addition to the Death Benefit under the Plan. Please refer to product brochure for further details and/or consult Relationship Manager.",
      "predicted": "In the policy, Extra Life Option provides an additional benefit of Extra Life Sum Assured upon death due to Accident. This means that if the policyholder dies due to an accident, they will receive not only the base sum assured but also an extra amount (up to the maximum allowed) as a separate benefit.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "Most key information is present in the predicted answer, but some minor details (e.g., 'Classic Benefit' and 'Refer to product brochure') are missing or not explicitly stated.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7913,
        0.7334,
        0.7293
      ],
      "avg_confidence": 0.7513
    },
    {
      "query": "How do I obtain a NOC from the bank if I have lost my Registration Certificate book for a vehicle under finance with M&N Bank",
      "expected": "You may obtain the NOC for a duplicate Registration Certificate book for your vehicle loan, by visiting any of our Retail Loan Service Center and submitting the following documents: Request form for applying for the required NOC. Self attested copy of the Registration Certificate book, duly self attested by you for validation. Copy of FIR or certificate from police or Acknowledgement from Police on written complaint with General Diary No. Applicable fee of Rs. 500//- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer,i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided. View more",
      "predicted": "You may obtain the NOC for a duplicate Registration Certificate book for your vehicle loan, by visiting any of our Retail Loan Service Center and submitting the following documents: Request form for applying for the required NOC. Self attested copy of the lost Registration Certificate book, duly self attested by you, along with the Original Registration Certificate book for validation. Applicable fee of Rs. 500/- vide a cheque/payorder/demand draft favouring M&N Bank Ltd Loan Account No. << >>. In case a bearer submits the request on your behalf, a bearer authorisation along with KYC documents(self attested copy and original for validation) of the bearer, i.e. Identity and Signature Proof like PAN Card, Aadhar Card, Driving License or Passport Copy will also need to be provided.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.9,
      "reason": "Most key information is present in both answers, with minor differences in formatting and wording.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.8565,
        0.8536,
        0.7733
      ],
      "avg_confidence": 0.8278
    },
    {
      "query": "Can I avail an Overdraft against my Property",
      "expected": "A Loan Against Property can be availed as a Term Loan with EMI repayments or as a Drop line Overdraft Facility.",
      "predicted": "No, you cannot avail an Overdraft against your Property. The context only mentions Loan Against Securities and Loan Against Property as two separate account options. There is no information about an Overdraft being available against a property.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "The predicted answer covers most of the key points in the expected answer, including Loan Against Property as a term loan or drop line overdraft facility.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7226,
        0.7002,
        0.6911
      ],
      "avg_confidence": 0.7046
    },
    {
      "query": "Where do I get this card",
      "expected": "ForexPlus card is available over the counter at M&N Bank branches. You simply need to walk into our branch with required documentation and walk out with ForexPlus card. The card will get activated within 4 hours from realization of funds. Alternatively, the customer can apply for card through NetBanking login also.In case the customer applies the card through NetBanking the card will be delivered to the customer within 2 working days. This facility is available at select locations only.",
      "predicted": "You can obtain the ForexPlus Chip card over the counter at M&N Bank branches. Simply walk into a branch with required documentation and you'll be able to take the card away with you. The activation process typically takes 4 hours from the realization of funds.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "Most key information is present in the predicted answer, but some minor details and formatting differences are missing or altered.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.705,
        0.7037,
        0.7011
      ],
      "avg_confidence": 0.7033
    },
    {
      "query": "Will I get an online confirmation of my transactions",
      "expected": "An order once placed will be confirmed immediately. The confirmation of units bought or sold will be done by the AMC / Registrar only on the next working day.",
      "predicted": "Yes, when you use Verified by Visa/ MasterCard SecureCode, you will receive a pop-up window on your computer screen with a Personal Assurance Message (PAM) or Personal Greeting message from M&N Bank, confirming the transaction.",
      "retrieval_success": false,
      "correct": true,
      "score": 0.8,
      "reason": "Most key information is present in both answers, but the predicted answer lacks specific details about AMC/Registrar confirmation.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7425,
        0.7208,
        0.6853
      ],
      "avg_confidence": 0.7162
    },
    {
      "query": "What are the documents required",
      "expected": "Please refer Policy wordings for exhaustive list of claims documents required. We will not ask for anything over and above the list given in Policy.",
      "predicted": "For opening a Current Account of a company, the following documents are required:\n\n1. Memorandum of Association (MOA)\n2. Articles of Association (AOA)\n3. Certificate of Incorporation\n4. Latest list of directors duly signed by any Director/Company Secretary /Authorised Signatory\n5. Board Resolution (BR) duly signed by the directors of the company.\n\nFor opening a Current Account of a partnership firm, the following documents are required:\n\n1. Latest Partnership Deed\n2. Documents establishing proof of existence\n3. Registration Certificate issued by Registrar of Firms\n4. Valid Business License or Certificate of Registration issued by State/Central Government authority\n5. PAN Card in the name of the firm\n6. Self-signed cheque issued from the firms account from a PSU/private sector/foreign bank along with a bank statement not older than four months\n7. Documents for establishing proof of address (such as landline telephone bill/electricity bill)\n8. Property Ownership Deed i.e. title deeds of the property in the name of the firm duly stamped and registered\n9. Latest property tax or water tax paid receipt/bill raised in the name of the firm\n10. TAN Allotment Letter in the name of the firm\n11. Existing bank account statement or passbook of a PSU/private sector/foreign bank\n\nFor opening a Current Account of a sole proprietorship firm, the following documents are required:\n\n1. Proof of existence in the name of firm\n2. Proof of address in the name of firm\n3. KYC of the proprietor\n4. Any two of the below listed documents for establishing proof of existence (such as registration certificate/license issued by Municipal authorities, etc.)",
      "retrieval_success": false,
      "correct": false,
      "score": 0.0,
      "reason": "Predicted does not match with expected",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "The response contains a detailed list of required documents for opening Current Accounts in different types of firms, which is not supported by the context."
      },
      "confidence_scores": [
        0.7192,
        0.7079,
        0.7067
      ],
      "avg_confidence": 0.7113
    },
    {
      "query": "How can I make a request to foreclose my Loans Against Rental Receivables account",
      "expected": "Kindly submit a written request to your nearest Retail Loan Service Centre to close your account. Our customer service desk will provide you with the necessary assistance.",
      "predicted": "I don't have information on that.",
      "retrieval_success": false,
      "correct": false,
      "score": 0.0,
      "reason": "Model returned a fallback response",
      "is_fallback": true,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Model does not know the answer"
      },
      "confidence_scores": [
        0.7408,
        0.7237,
        0.6976
      ],
      "avg_confidence": 0.7207
    },
    {
      "query": "Will I be able to access both my accounts at all establishments/ATMs through a single Debit Card",
      "expected": "Your existing Debit Card will be linked to both accounts. Any additional card will be charged at the regular rates. You will be able to select either account to transact at M&N Bank ATMs. At other VISA/PLUS ATMs (Other Banks ATMs) you may or may not have access to both accounts. When such an option is unavailable, you can only access your salary account. If you are a VISA Electron Debit Cardholder you can access only your Salary Account at merchant locations for transactions and not your Reimbursement Account. If you are a Maestro Debit Cardholder may be able to select between your Salary Account and Reimbursement Account at merchant locations.",
      "predicted": "Yes, as a joint Super Saver Account holder, you can get an ATM / Debit Card that allows access to both accounts at most establishments/ATMs.",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "The predicted answer covers most of the key points in the expected answer, but lacks specificity about VISA/PLUS ATMs and merchant locations.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.6918,
        0.6886,
        0.6799
      ],
      "avg_confidence": 0.6868
    },
    {
      "query": "How do I know my application has been accepted",
      "expected": "You can check your account opening status on www.M&N sec.com on the footer page of the homepage section page under the Application status.",
      "predicted": "You will be informed once your application is accepted and all the required accounts are set up. In case your application is not processed because of lack of some details, our representative will contact you by mail.",
      "retrieval_success": false,
      "correct": true,
      "score": 0.8,
      "reason": "The predicted answer covers most of the key points in the expected answer but lacks specific details about checking account status online.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7143,
        0.68,
        0.6228
      ],
      "avg_confidence": 0.6724
    },
    {
      "query": "How do Recurring Deposits work",
      "expected": "A monthly deposit of a pre-fixed amount is made in the Recurring Deposit account. On maturity, you get back the amount you deposited, along with interest calculated at the rate applicable at the time of opening your account.",
      "predicted": "Recurring Deposits can now be booked through NetBanking. Following are the steps for booking your Recurring Deposit online: Access your NetBanking account with your Customer ID and IPIN (NetBanking Password) Select Open Recurring Deposit option under the Recurring Deposit menu from the Menu bar located on the left hand side of the web page Select the desired values from the Drop-down lists Once complete, click on Continue and Confirm the details entered",
      "retrieval_success": true,
      "correct": true,
      "score": 0.8,
      "reason": "The predicted answer covers most of the key points in the expected answer, but lacks specific details about interest calculation and maturity amount.",
      "is_fallback": false,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Response is correct"
      },
      "confidence_scores": [
        0.7399,
        0.7276,
        0.7272
      ],
      "avg_confidence": 0.7316
    },
    {
      "query": "How much am I charged for this service",
      "expected": "A quarterly maintenance charge is applicable to hold an Investment Services Account. We request you to contact your RM / Personal Banker or nearest M&N Bank Branch for details.",
      "predicted": "I don't have information on that.",
      "retrieval_success": false,
      "correct": false,
      "score": 0.0,
      "reason": "Model returned a fallback response",
      "is_fallback": true,
      "hallucination_detail": {
        "hallucinated": false,
        "confidence": "high",
        "reason": "Model does not know the answer"
      },
      "confidence_scores": [
        0.6953,
        0.6919,
        0.6699
      ],
      "avg_confidence": 0.6857
    }
  ]
}