evaluation/dataset.json

[
  {
    "id": "case_1",
    "question": "Why was application APP-78432 flagged as high risk?",
    "expected_intent": "Analyze why application APP-78432 was flagged",
    "expected_answer_key_points": [
      "Fraud Score: 449",
      "Risk Level: MEDIUM",
      "Model: XGBoost Fraud Ensemble v3.2",
      "Decision: APPROVED"
    ]
  },
  {
    "id": "case_2",
    "question": "Check fair lending compliance for APP-55555",
    "expected_intent": "Check fair lending compliance",
    "expected_answer_key_points": [
      "No geographic, name-based, or age-related discrimination",
      "Adverse Impact Ratio: 0.94",
      "Status: COMPLIANT"
    ]
  },
  {
    "id": "case_3",
    "question": "Explain the fraud score for APP-12345 and compare it to approved applications",
    "expected_intent": "Explain fraud score and compare to population",
    "expected_answer_key_points": [
      "Fraud Score: 850",
      "Risk Level: HIGH",
      "Suspicious Pattern: High velocity of applications",
      "Identity verification usage"
    ]
  },
  {
    "id": "synth_4_synth_high_risk_1",
    "question": "Why is this account flagged as high risk for synthetic identity fraud?",
    "expected_intent": "Understand the reasons behind the high-risk classification.",
    "expected_answer_key_points": [
      "Unusual account activity patterns",
      "Mismatch between identity attributes",
      "Use of known fraudulent identifiers"
    ]
  },
  {
    "id": "synth_5_synth_high_risk_2",
    "question": "What specific behaviors in the transaction history indicate synthetic identity fraud?",
    "expected_intent": "Identify behaviors that suggest synthetic identity fraud.",
    "expected_answer_key_points": [
      "Multiple transactions with inconsistent locations",
      "Rapid account activity after dormancy",
      "Attempts to access high-value services"
    ]
  },
  {
    "id": "synth_6_synth_high_risk_3",
    "question": "How does the model differentiate between synthetic identities and legitimate customers?",
    "expected_intent": "Understand the model's methodology for distinguishing between synthetic and legitimate identities.",
    "expected_answer_key_points": [
      "Advanced pattern recognition algorithms",
      "Comparison against known legitimate customer profiles",
      "Analysis of identity verification documentation"
    ]
  },
  {
    "id": "synth_7_synth_high_risk_4",
    "question": "What are the common data points used by the model to detect synthetic identities?",
    "expected_intent": "Identify key data points used by the model for detection.",
    "expected_answer_key_points": [
      "Social security number validation",
      "Cross-referencing with public records",
      "Analysis of digital footprint and device information"
    ]
  },
  {
    "id": "synth_8_low_risk_fp_1",
    "question": "Why was this transaction flagged as fraudulent despite being low risk?",
    "expected_intent": "Understand the reasons behind a false positive classification.",
    "expected_answer_key_points": [
      "Unusual transaction pattern",
      "Customer purchase history",
      "Changes in location or device used"
    ]
  },
  {
    "id": "synth_9_low_risk_fp_2",
    "question": "Can you explain why this account's activity is considered suspicious?",
    "expected_intent": "Gain insights into the factors influencing the fraud model's decision.",
    "expected_answer_key_points": [
      "High volume of transactions",
      "Transaction value deviations",
      "Comparison with typical user behavior"
    ]
  },
  {
    "id": "synth_10_low_risk_fp_3",
    "question": "What factors led to this low-risk transaction being marked as fraud?",
    "expected_intent": "Identify specific elements that triggered the fraud alert.",
    "expected_answer_key_points": [
      "Mismatch in expected transaction time",
      "Discrepancy in user verification",
      "Recent account changes or updates"
    ]
  },
  {
    "id": "synth_11_low_risk_fp_4",
    "question": "Why did our system mistakenly flag this legitimate transaction?",
    "expected_intent": "Determine the cause of the system error leading to a false positive.",
    "expected_answer_key_points": [
      "Error in model threshold setting",
      "Anomaly detection misclassification",
      "Lack of data on new customer behavior"
    ]
  },
  {
    "id": "synth_12_synth_borderline_case_1",
    "question": "Why is this transaction flagged as suspicious when the customer has a long history of legitimate purchases?",
    "expected_intent": "Understand mixed signals causing a fraud alert.",
    "expected_answer_key_points": [
      "Analyzing recent transaction behavior",
      "Comparing with customer's purchase history",
      "Highlighting unusual transaction patterns"
    ]
  },
  {
    "id": "synth_13_synth_borderline_case_2",
    "question": "Can you explain why this low amount transaction is considered high-risk while the customer has been verified recently?",
    "expected_intent": "Clarify factors contributing to risk evaluation.",
    "expected_answer_key_points": [
      "Risk assessment includes transaction context",
      "Verification status vs. transaction attributes",
      "Analysis of current vs. past behavior"
    ]
  },
  {
    "id": "synth_14_synth_borderline_case_3",
    "question": "This merchant is reputable, so why are some of their transactions flagged for potential fraud?",
    "expected_intent": "Investigate reasons behind inconsistent fraud signals.",
    "expected_answer_key_points": [
      "Merchant transaction volume vs. individual transactions",
      "Potential changes in merchant activities",
      "Flagged patterns specific to current transactions"
    ]
  },
  {
    "id": "synth_15_synth_borderline_case_4",
    "question": "Why is this account flagged when the user is known for frequent travel and varied spending patterns?",
    "expected_intent": "Identify causes of false positives in fraud detection.",
    "expected_answer_key_points": [
      "Account flagged due to recent activity anomalies",
      "Analysis of travel-related spending patterns",
      "Consideration of geographic and spending behavior"
    ]
  },
  {
    "id": "synth_16_fair_lending_compliance_1",
    "question": "Why was the loan application of a minority applicant flagged as high risk by the model?",
    "expected_intent": "Understand the factors leading to the high-risk categorization for minority applicants.",
    "expected_answer_key_points": [
      "Model's risk assessment criteria",
      "Applicant's credit history and income",
      "Any bias or unfair treatment in the model"
    ]
  },
  {
    "id": "synth_17_fair_lending_compliance_2",
    "question": "Can you explain if the model's decision impacts applicants from low-income neighborhoods differently?",
    "expected_intent": "Determine if there is any disparate impact based on the applicant's neighborhood.",
    "expected_answer_key_points": [
      "Impact of location on risk scoring",
      "Comparison of approval rates by neighborhood",
      "Fair lending compliance measures in place"
    ]
  },
  {
    "id": "synth_18_fair_lending_compliance_3",
    "question": "What measures are in place to ensure that the model's decisions comply with fair lending regulations?",
    "expected_intent": "Identify compliance protocols for fair lending regulations within the model.",
    "expected_answer_key_points": [
      "Regular bias audits",
      "Use of fair lending algorithms",
      "Continuous monitoring for compliance"
    ]
  },
  {
    "id": "synth_19_fair_lending_compliance_4",
    "question": "How does the model ensure equal treatment for applicants with similar financial profiles but from different demographic groups?",
    "expected_intent": "Verify the model's fairness and equality in decision-making across demographics.",
    "expected_answer_key_points": [
      "Demographic parity in decision-making",
      "Comparative analysis of similar profiles",
      "Adjustments made to mitigate bias"
    ]
  },
  {
    "id": "synth_20_synth_high_risk_1",
    "question": "Why is the model flagging an unusually high number of transactions as high-risk this month?",
    "expected_intent": "Understand the reason behind the spike in high-risk transaction flags.",
    "expected_answer_key_points": [
      "Change in transaction patterns",
      "Adjustment in model thresholds",
      "New data inputs affecting model behavior"
    ]
  },
  {
    "id": "synth_21_synth_low_accuracy_2",
    "question": "What could be causing the recent drop in model accuracy for identifying fraudulent transactions?",
    "expected_intent": "Identify factors leading to decreased model accuracy.",
    "expected_answer_key_points": [
      "Data drift or changes in data distribution",
      "Outdated model parameters",
      "Need for model retraining"
    ]
  },
  {
    "id": "synth_22_synth_segment_discrepancy_3",
    "question": "Why is the model performing poorly on certain customer segments?",
    "expected_intent": "Explain performance discrepancies across customer segments.",
    "expected_answer_key_points": [
      "Segment-specific behavior not well-captured",
      "Imbalanced training data for segments",
      "Potential need for segment-specific features"
    ]
  },
  {
    "id": "synth_23_synth_feature_importance_4",
    "question": "What are the most influential features the model uses to determine fraud risk?",
    "expected_intent": "Identify key features driving model predictions.",
    "expected_answer_key_points": [
      "List of top contributing features",
      "Feature importance ranking",
      "Impact of each feature on the model's decision"
    ]
  }
]