data/eval_v2/eval.jsonl

{"question": "How many days of annual paid leave do EU employees get?", "relevant_doc_ids": ["vacation-eu"], "expected_answer": "30 working days.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "vacation-eu", "span": "receive 30 working days of annual paid leave"}]}
{"question": "Until when must leftover holiday days from last year be used up in Germany?", "relevant_doc_ids": ["vacation-eu"], "expected_answer": "By 31 March, after which they expire.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "vacation-eu", "span": "must be taken by 31 March"}]}
{"question": "I am based in Austin with four years at the company, how many days off do I get each year?", "relevant_doc_ids": ["vacation-us"], "expected_answer": "25 days of paid time off, because the allowance rises to 25 after three years of service.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "vacation-us", "span": "rising to 25 days after three years of service"}]}
{"question": "Can I get paid out for unused holiday days in Pune?", "relevant_doc_ids": ["vacation-india"], "expected_answer": "Yes, up to 12 unused days can be encashed in December at the basic daily rate, but not both encashed and carried forward.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "vacation-india", "span": "Up to 12 unused days may be encashed in December"}]}
{"question": "How long can the non-birthing parent stay home on full pay?", "relevant_doc_ids": ["parental-leave"], "expected_answer": "6 weeks as a secondary caregiver.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "parental-leave", "span": "secondary caregivers receive 6 weeks"}]}
{"question": "How many paid sick days do employees get per year?", "relevant_doc_ids": ["sick-leave-global"], "expected_answer": "10 paid sick days.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "sick-leave-global", "span": "10 paid sick days per year"}]}
{"question": "What is the vacation allowance for contractors based in Brazil?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How often do engineers get a new laptop?", "relevant_doc_ids": ["laptop-refresh-engineering"], "expected_answer": "Every 36 months.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "laptop-refresh-engineering", "span": "every 36 months"}]}
{"question": "I work in customer success, when am I due a replacement computer?", "relevant_doc_ids": ["laptop-refresh-sales"], "expected_answer": "Every 48 months.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "laptop-refresh-sales", "span": "every 48 months"}]}
{"question": "How much money do I get towards a second screen?", "relevant_doc_ids": ["monitor-stipend"], "expected_answer": "A one-time stipend of 300 euros for an external monitor.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "monitor-stipend", "span": "one-time stipend of 300 euros"}]}
{"question": "Is there a yearly payment for internet and electricity if I never go into the office?", "relevant_doc_ids": ["home-office-grant"], "expected_answer": "Yes, a 600 euro annual grant paid in January and July.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "home-office-grant", "span": "annual grant of 600 euros for internet, electricity and coworking costs"}]}
{"question": "How long are device disposal certificates kept?", "relevant_doc_ids": ["device-disposal"], "expected_answer": "7 years.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "device-disposal", "span": "stored for 7 years"}]}
{"question": "After I receive my new engineering laptop, what happens to the old one and how is it handled afterwards?", "relevant_doc_ids": ["laptop-refresh-engineering", "device-disposal"], "expected_answer": "It must be returned to IT within 14 days, then it is wiped with a certified three-pass erasure and recycled.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "laptop-refresh-engineering", "span": "must be returned to IT within 14 days"}, {"doc_id": "device-disposal", "span": "wiped with a certified three-pass erasure"}]}
{"question": "What is the policy on reimbursing personal phone roaming charges?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "What is the meal per-diem on domestic trips?", "relevant_doc_ids": ["travel-domestic"], "expected_answer": "40 euros per travel day.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "travel-domestic", "span": "Per-diem for meals is 40 euros per travel day"}]}
{"question": "How far ahead do I need sign-off for a trip abroad?", "relevant_doc_ids": ["travel-international"], "expected_answer": "At least 14 days before departure, from the department head.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "travel-international", "span": "at least 14 days before departure"}]}
{"question": "How much do technicians get for food while installing robots at a customer warehouse?", "relevant_doc_ids": ["meal-allowance-onsite"], "expected_answer": "25 euros per day, if the customer does not provide meals.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "meal-allowance-onsite", "span": "meal allowance of 25 euros per day"}]}
{"question": "What is the most I can spend per head on a customer dinner?", "relevant_doc_ids": ["client-entertainment"], "expected_answer": "80 euros per person.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "client-entertainment", "span": "80 euros per person for a dinner"}]}
{"question": "How long after buying something can I still file the claim?", "relevant_doc_ids": ["expense-reimbursement-deadline"], "expected_answer": "45 days from the purchase date.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "expense-reimbursement-deadline", "span": "within 45 days of the purchase date"}]}
{"question": "How much can I claim per day for meals on a trip abroad, and how soon must I submit the claim?", "relevant_doc_ids": ["travel-international", "expense-reimbursement-deadline"], "expected_answer": "A per-diem of 65 euros per travel day, with the claim submitted within 45 days of the purchase date.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "travel-international", "span": "per-diem of 65 euros per travel day"}, {"doc_id": "expense-reimbursement-deadline", "span": "within 45 days of the purchase date"}]}
{"question": "Are hotel minibar charges reimbursable?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "When does the platform on-call rotation change over?", "relevant_doc_ids": ["oncall-rotation-platform"], "expected_answer": "Every Monday at 10:00 UTC.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "oncall-rotation-platform", "span": "changes every Monday at 10:00 UTC"}]}
{"question": "How fast must the data team on-call engineer acknowledge a page?", "relevant_doc_ids": ["oncall-rotation-data"], "expected_answer": "Within 20 minutes.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "oncall-rotation-data", "span": "must acknowledge a page within 20 minutes"}]}
{"question": "What is the base pay for carrying the pager for a week?", "relevant_doc_ids": ["oncall-stipend"], "expected_answer": "A flat weekly stipend of 300 euros.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "oncall-stipend", "span": "flat weekly stipend of 300 euros"}]}
{"question": "How much bonus do I earn if I am paged because checkout is completely down?", "relevant_doc_ids": ["oncall-stipend", "incident-severity-levels"], "expected_answer": "A complete checkout outage is Sev1, which pays a 120 euro bonus if the engineer joins the call within the response time.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "oncall-stipend", "span": "each page that is classified as Sev1 pays a bonus of 120 euros"}, {"doc_id": "incident-severity-levels", "span": "complete outage of a customer-facing service such as fleet control, checkout"}]}
{"question": "Who gets paged if nobody answers for half an hour?", "relevant_doc_ids": ["escalation-policy"], "expected_answer": "The VP of Engineering.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "escalation-policy", "span": "finally to the VP of Engineering after 30 minutes without acknowledgement"}]}
{"question": "What is the response time for a Sev2 incident?", "relevant_doc_ids": ["incident-severity-levels"], "expected_answer": "30 minutes.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "incident-severity-levels", "span": "with a 30-minute response time"}]}
{"question": "Who runs the incident when checkout goes down, and how fast must the page be answered?", "relevant_doc_ids": ["escalation-policy", "incident-severity-levels"], "expected_answer": "An engineering manager is the incident commander, and the response time for Sev1 is 10 minutes.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "escalation-policy", "span": "The incident commander for a Sev1 is always an engineering manager"}, {"doc_id": "incident-severity-levels", "span": "response time is 10 minutes"}]}
{"question": "What is the on-call rotation schedule for the support team?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How long does failing payments over to the standby region take?", "relevant_doc_ids": ["runbook-payments-outage"], "expected_answer": "About 4 minutes.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "runbook-payments-outage", "span": "which takes about 4 minutes"}]}
{"question": "Who must give written approval before the ledger database is restarted during an outage?", "relevant_doc_ids": ["runbook-payments-outage"], "expected_answer": "The finance director.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "runbook-payments-outage", "span": "needs written approval from the finance director"}]}
{"question": "Sign-in is failing for customers, what is the first action the on-call should take?", "relevant_doc_ids": ["runbook-auth-outage"], "expected_answer": "Roll back the most recent auth deployment before investigating.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "runbook-auth-outage", "span": "rolls back the most recent auth deployment before investigating"}]}
{"question": "At what backlog size should we add more indexing workers?", "relevant_doc_ids": ["runbook-search-degraded"], "expected_answer": "When the indexer queue exceeds 50,000 messages, scale workers from 4 to 12.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "runbook-search-degraded", "span": "if it exceeds 50,000 messages, scales indexer workers from 4 to 12"}]}
{"question": "Within how many business days must a postmortem be published?", "relevant_doc_ids": ["postmortem-template"], "expected_answer": "Within 5 business days.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "postmortem-template", "span": "must be published within 5 business days"}]}
{"question": "Does a minor degradation with a workaround need a written postmortem?", "relevant_doc_ids": ["postmortem-template", "incident-severity-levels"], "expected_answer": "No, a postmortem is only required for Sev1 and Sev2, and a minor degradation with a workaround is Sev3.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "postmortem-template", "span": "required for every Sev1 and Sev2 incident"}, {"doc_id": "incident-severity-levels", "span": "Sev3 is a minor degradation with a workaround"}]}
{"question": "How quickly does a full outage appear on the public status page?", "relevant_doc_ids": ["status-page-policy"], "expected_answer": "A Sev1 is posted within 15 minutes of declaration.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "status-page-policy", "span": "A Sev1 is posted within 15 minutes of declaration"}]}
{"question": "What is the runbook for a robot telemetry outage?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "When does the legacy v1 fleet software support end?", "relevant_doc_ids": ["release-v1-legacy"], "expected_answer": "30 June 2027.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "release-v1-legacy", "span": "Support for v1 ends on 30 June 2027"}]}
{"question": "What share of traffic gets a new v2 build during canary and for how long?", "relevant_doc_ids": ["release-v2-current"], "expected_answer": "5 percent of traffic for 30 minutes.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "release-v2-current", "span": "Canary deployments serve 5 percent of traffic for 30 minutes"}]}
{"question": "Who has to approve a change that skips the normal freeze to fix an outage?", "relevant_doc_ids": ["hotfix-process"], "expected_answer": "One engineering manager and one member of the release team.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "hotfix-process", "span": "needs approval from one engineering manager and one member of the release team"}]}
{"question": "What is the longest a feature toggle is allowed to live?", "relevant_doc_ids": ["feature-flag-policy"], "expected_answer": "90 days.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "feature-flag-policy", "span": "maximum lifetime of 90 days"}]}
{"question": "How long does a rollback of a v2 deployment take?", "relevant_doc_ids": ["rollback-procedure"], "expected_answer": "About 3 minutes.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "rollback-procedure", "span": "within about 3 minutes"}]}
{"question": "Can a hotfix be deployed on a Friday evening, and does it need a canary stage?", "relevant_doc_ids": ["hotfix-process", "release-v2-current"], "expected_answer": "Yes, hotfixes are exempt from the Friday pause on v2, and for a Sev1 the canary stage may be skipped.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "release-v2-current", "span": "Deployments are paused on Fridays after 14:00 UTC and during the last week of each quarter, except for hotfixes"}, {"doc_id": "hotfix-process", "span": "may skip the canary stage for Sev1"}]}
{"question": "Who approves a change to the legacy v0 firmware release schedule?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How many robots does the Starter plan support?", "relevant_doc_ids": ["plan-starter"], "expected_answer": "Up to 25 robots in a single warehouse site.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "plan-starter", "span": "supports up to 25 robots in a single warehouse site"}]}
{"question": "Which plan has single sign-on through SAML and a limit of 600 API requests per minute?", "relevant_doc_ids": ["plan-pro"], "expected_answer": "The Pro plan.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "plan-pro", "span": "single sign-on through SAML, custom reports and an API limit of 600 requests per minute"}]}
{"question": "What is the minimum contract length for the top tier?", "relevant_doc_ids": ["plan-enterprise"], "expected_answer": "A minimum of 24 months.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "plan-enterprise", "span": "minimum of 24 months"}]}
{"question": "How much does the predictive maintenance module cost per month?", "relevant_doc_ids": ["plan-addons"], "expected_answer": "900 euros per month.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "plan-addons", "span": "predictive maintenance module costs 900 euros per month"}]}
{"question": "Does a university get a discount on an Enterprise contract?", "relevant_doc_ids": ["plan-discount-nonprofit"], "expected_answer": "No, the 30 percent discount applies only to Starter and Pro plans.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "plan-discount-nonprofit", "span": "does not apply to add-ons or to Enterprise contracts"}]}
{"question": "What is the monthly cost of a Pro customer who wants the simulation sandbox?", "relevant_doc_ids": ["plan-pro", "plan-addons"], "expected_answer": "The Pro plan is 7,900 euros per month and the sandbox is an extra 600 euros per month, so 8,500 euros.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "plan-pro", "span": "costs 7,900 euros per month"}, {"doc_id": "plan-addons", "span": "costs 600 euros per month on Pro"}]}
{"question": "What does a registered charity pay per month for the Pro plan?", "relevant_doc_ids": ["plan-pro", "plan-discount-nonprofit"], "expected_answer": "30 percent off 7,900 euros, which is 5,530 euros per month.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "plan-pro", "span": "costs 7,900 euros per month"}, {"doc_id": "plan-discount-nonprofit", "span": "receive a 30 percent discount on Starter and Pro plans"}]}
{"question": "Does the company offer a free trial of the fleet software?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "What is the first response target for Starter support tickets?", "relevant_doc_ids": ["sla-starter"], "expected_answer": "8 business hours.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "sla-starter", "span": "first response target is 8 business hours"}]}
{"question": "Which plan offers 99.9 percent uptime and a service credit capped at 50 percent?", "relevant_doc_ids": ["sla-pro"], "expected_answer": "The Pro plan.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "sla-pro", "span": "capped at 50 percent"}]}
{"question": "How quickly does the top tier answer a critical ticket in the middle of the night?", "relevant_doc_ids": ["sla-enterprise"], "expected_answer": "Within 30 minutes at any hour.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "sla-enterprise", "span": "Critical tickets receive a response within 30 minutes at any hour"}]}
{"question": "Who answers Pro tickets from Singapore after the Pune team goes offline?", "relevant_doc_ids": ["support-hours-apac"], "expected_answer": "The Frankfurt team.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "support-hours-apac", "span": "Pro tickets from Asia Pacific are routed to the Frankfurt team"}]}
{"question": "What word can a customer reply with to escalate a ticket?", "relevant_doc_ids": ["support-escalation"], "expected_answer": "ESCALATE.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "support-escalation", "span": "replying with the word ESCALATE"}]}
{"question": "A Pro customer paying the list price got 99.7 percent uptime, how much service credit do they receive?", "relevant_doc_ids": ["sla-pro", "plan-pro"], "expected_answer": "99.7 is 0.2 points below the 99.9 target, so 20 percent of the 7,900 euro monthly fee, which is 1,580 euros.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "sla-pro", "span": "service credit of 10 percent of the monthly fee for each 0.1 percentage point below target"}, {"doc_id": "plan-pro", "span": "costs 7,900 euros per month"}]}
{"question": "What is the uptime commitment for the free trial tier?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "What is the minimum password length?", "relevant_doc_ids": ["password-policy"], "expected_answer": "14 characters.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "password-policy", "span": "at least 14 characters"}]}
{"question": "Are one-time codes by SMS accepted for logging in to any system?", "relevant_doc_ids": ["mfa-policy"], "expected_answer": "No, text message codes are not accepted for any system.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "mfa-policy", "span": "text message codes are not accepted for any system"}]}
{"question": "How often do production database credentials change?", "relevant_doc_ids": ["secrets-management"], "expected_answer": "Every 30 days.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "secrets-management", "span": "production database credentials rotate every 30 days"}]}
{"question": "What do we do when an API key is accidentally committed to a repository?", "relevant_doc_ids": ["secrets-management"], "expected_answer": "Treat it as compromised and revoke it within 1 hour, then clean the commit history.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "secrets-management", "span": "the secret is revoked within 1 hour"}]}
{"question": "Which report is required from vendors handling robot telemetry?", "relevant_doc_ids": ["vendor-security-review"], "expected_answer": "A current SOC 2 Type II report.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "vendor-security-review", "span": "a current SOC 2 Type II report"}]}
{"question": "How quickly must I tell IT if my laptop is stolen?", "relevant_doc_ids": ["laptop-encryption"], "expected_answer": "Within 4 hours.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "laptop-encryption", "span": "reported to IT within 4 hours"}]}
{"question": "I lost my hardware security key and my laptop on the same trip, how fast must each loss be reported?", "relevant_doc_ids": ["mfa-policy", "laptop-encryption"], "expected_answer": "The key must be reported to the security team within 24 hours and the laptop to IT within 4 hours.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "mfa-policy", "span": "must be reported to the security team within 24 hours"}, {"doc_id": "laptop-encryption", "span": "reported to IT within 4 hours"}]}
{"question": "What is the policy on using personal phones for work email?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How long are audit logs of customer data access kept?", "relevant_doc_ids": ["retention-logs"], "expected_answer": "7 years.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "retention-logs", "span": "retained for 7 years"}]}
{"question": "How long is raw robot sensor data stored before it is summarised?", "relevant_doc_ids": ["retention-telemetry"], "expected_answer": "90 days.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "retention-telemetry", "span": "stored for 90 days"}]}
{"question": "How long do we keep contract records after a customer leaves?", "relevant_doc_ids": ["retention-customer-data"], "expected_answer": "For the length of the contract plus 6 years.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "retention-customer-data", "span": "the length of the contract plus 6 years"}]}
{"question": "By when must a request to erase someone's personal data be finished?", "relevant_doc_ids": ["gdpr-deletion-request"], "expected_answer": "Within 30 days, after being acknowledged within 5 days.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "gdpr-deletion-request", "span": "completed within 30 days"}]}
{"question": "What is the recovery time objective for the core platform?", "relevant_doc_ids": ["backup-policy"], "expected_answer": "4 hours.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "backup-policy", "span": "recovery time objective is 4 hours"}]}
{"question": "If a customer asks to be erased, how long can their data still exist in backups?", "relevant_doc_ids": ["gdpr-deletion-request", "backup-policy"], "expected_answer": "Backups are not edited; deleted records disappear when the backup expires after 35 days.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "gdpr-deletion-request", "span": "deleted records are removed when the backup expires after 35 days"}, {"doc_id": "backup-policy", "span": "point-in-time recovery for 35 days"}]}
{"question": "A customer terminates their contract, how long until their site telemetry is gone and how long are their contract records kept?", "relevant_doc_ids": ["retention-telemetry", "retention-customer-data"], "expected_answer": "Telemetry is deleted within 60 days of termination; contract records are kept for the contract length plus 6 years.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "retention-telemetry", "span": "deleted within 60 days of contract termination"}, {"doc_id": "retention-customer-data", "span": "the length of the contract plus 6 years"}]}
{"question": "How long are recordings of customer support calls retained?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How many interviewers must agree for an engineering hire?", "relevant_doc_ids": ["interview-process-engineering"], "expected_answer": "At least three of the four interviewers.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "interview-process-engineering", "span": "agreement from at least three of the four interviewers"}]}
{"question": "Is there a coding exercise when hiring for sales?", "relevant_doc_ids": ["interview-process-sales"], "expected_answer": "No, sales candidates do a role-play instead.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "interview-process-sales", "span": "There is no coding exercise"}]}
{"question": "How soon after joining is an engineer expected to push their first change to production?", "relevant_doc_ids": ["onboarding-week-one"], "expected_answer": "Within their first 5 working days.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "onboarding-week-one", "span": "within their first 5 working days"}]}
{"question": "How much do mentors receive for finishing the new-hire pairing programme?", "relevant_doc_ids": ["buddy-program"], "expected_answer": "A recognition bonus of 150 euros.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "buddy-program", "span": "recognition bonus of 150 euros"}]}
{"question": "How much is the reward if I refer a senior engineer who gets hired?", "relevant_doc_ids": ["referral-bonus"], "expected_answer": "4,000 euros, paid half after 3 months and half after 12.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "referral-bonus", "span": "4,000 euros for senior engineering roles"}]}
{"question": "If I refer a senior engineer and they are hired, when do I receive the first part of my reward and is a take-home exercise likely?", "relevant_doc_ids": ["interview-process-engineering", "referral-bonus"], "expected_answer": "The first half is paid after the new hire completes 3 months, and senior roles get a take-home exercise.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "interview-process-engineering", "span": "Candidates are given a take-home exercise only for senior roles"}, {"doc_id": "referral-bonus", "span": "the first after the new hire completes 3 months"}]}
{"question": "When do managers have to submit ratings?", "relevant_doc_ids": ["review-cycle"], "expected_answer": "By 15 December.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "review-cycle", "span": "Managers submit ratings by 15 December"}]}
{"question": "How many review cycles of good performance are needed before an engineer can be promoted?", "relevant_doc_ids": ["promotion-criteria-ic"], "expected_answer": "At least two consecutive review cycles.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "promotion-criteria-ic", "span": "at least two consecutive review cycles"}]}
{"question": "How long must someone have been a manager before going up to senior manager?", "relevant_doc_ids": ["promotion-criteria-manager"], "expected_answer": "At least 18 months.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "promotion-criteria-manager", "span": "at least 18 months in the current management role"}]}
{"question": "What fraction of a department can be given the best possible rating?", "relevant_doc_ids": ["calibration"], "expected_answer": "No more than 10 percent.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "calibration", "span": "No more than 10 percent of a department may receive the top rating"}]}
{"question": "How long does a performance improvement plan last?", "relevant_doc_ids": ["pip-process"], "expected_answer": "60 days.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "pip-process", "span": "lasts 60 days"}]}
{"question": "Ratings are shared in January, when is the next time promotions are announced and when does a promotion take effect?", "relevant_doc_ids": ["review-cycle", "promotion-criteria-ic"], "expected_answer": "Promotions are announced in February and August and take effect on the first day of the following month.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "review-cycle", "span": "results are shared with employees in the first week of January"}, {"doc_id": "promotion-criteria-ic", "span": "announced twice a year, in February and August"}]}
{"question": "What is the bonus target for senior managers?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "What payload can the H200 carry?", "relevant_doc_ids": ["robot-model-h200"], "expected_answer": "Up to 35 kilograms.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "robot-model-h200", "span": "payload of up to 35 kilograms"}]}
{"question": "Which robot needs aisles of at least two metres?", "relevant_doc_ids": ["robot-model-h400"], "expected_answer": "The H400.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "robot-model-h400", "span": "requires aisles of at least 2.0 metres"}]}
{"question": "How long does it take a technician to change a battery without switching the robot off?", "relevant_doc_ids": ["battery-swap"], "expected_answer": "Under 90 seconds.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "battery-swap", "span": "in under 90 seconds"}]}
{"question": "When is a battery taken out of service?", "relevant_doc_ids": ["battery-swap"], "expected_answer": "After 1,200 charge cycles or when capacity drops below 70 percent, whichever comes first.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "battery-swap", "span": "retired after 1,200 charge cycles"}]}
{"question": "How long does an H400 need to get to 80 percent charge?", "relevant_doc_ids": ["charging-station"], "expected_answer": "About 85 minutes.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "charging-station", "span": "85 minutes for the H400"}]}
{"question": "How often do robots get a full service?", "relevant_doc_ids": ["maintenance-schedule"], "expected_answer": "Every 2,000 operating hours.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "maintenance-schedule", "span": "A full service every 2,000 operating hours"}]}
{"question": "How long does an H200 run on one charge, and when must its battery eventually be retired?", "relevant_doc_ids": ["robot-model-h200", "battery-swap"], "expected_answer": "About 8 hours of continuous operation; retired after 1,200 charge cycles or below 70 percent capacity.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "robot-model-h200", "span": "about 8 hours of continuous operation"}, {"doc_id": "battery-swap", "span": "retired after 1,200 charge cycles"}]}
{"question": "How many charging stations does a 40 robot site need at minimum, and how often are robots inspected?", "relevant_doc_ids": ["charging-station", "maintenance-schedule"], "expected_answer": "At least one station per four robots, so 10 stations; a basic inspection every 500 operating hours.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "charging-station", "span": "one station for every four robots as a minimum"}, {"doc_id": "maintenance-schedule", "span": "basic inspection every 500 operating hours"}]}
{"question": "What is the maximum payload of the H600 model?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How fast do robots stop when the local pause button is pressed?", "relevant_doc_ids": ["safety-zones"], "expected_answer": "Within 2 seconds.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "safety-zones", "span": "within 2 seconds"}]}
{"question": "How quickly do the brakes engage after an emergency stop button is hit?", "relevant_doc_ids": ["emergency-stop"], "expected_answer": "In less than 0.5 seconds.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "emergency-stop", "span": "in less than 0.5 seconds"}]}
{"question": "How long can a customer postpone a routine firmware release?", "relevant_doc_ids": ["firmware-update-policy"], "expected_answer": "Up to 60 days; security updates cannot be deferred.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "firmware-update-policy", "span": "defer a non-critical update by up to 60 days"}]}
{"question": "Within what time must a near miss be reported?", "relevant_doc_ids": ["near-miss-reporting"], "expected_answer": "Within 48 hours.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "near-miss-reporting", "span": "within 48 hours"}]}
{"question": "When do I need a hard hat in a robot zone?", "relevant_doc_ids": ["ppe-requirements"], "expected_answer": "Only where goods are stored above 4 metres or in H400 pallet areas.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "ppe-requirements", "span": "Hard hats are required only where goods are stored above 4 metres"}]}
{"question": "A robot clips a pallet and a staff member presses the emergency stop, what must be logged and how quickly?", "relevant_doc_ids": ["emergency-stop", "near-miss-reporting"], "expected_answer": "The emergency stop must be logged within 24 hours with its cause, and the near miss reported within 48 hours.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "emergency-stop", "span": "must be logged within 24 hours, with its cause"}, {"doc_id": "near-miss-reporting", "span": "within 48 hours"}]}
{"question": "Who pays for prescription safety glasses?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "What HTTP status does the fleet API return when the rate limit is exceeded?", "relevant_doc_ids": ["fleet-api-rate-limits"], "expected_answer": "429 with a Retry-After header.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "fleet-api-rate-limits", "span": "returns status 429"}]}
{"question": "How many times is a failed webhook delivery retried?", "relevant_doc_ids": ["fleet-webhooks"], "expected_answer": "Up to 8 times over 24 hours.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "fleet-webhooks", "span": "retried up to 8 times"}]}
{"question": "How long does an access token for the fleet API remain valid?", "relevant_doc_ids": ["fleet-auth-tokens"], "expected_answer": "1 hour.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "fleet-auth-tokens", "span": "Access tokens expire after 1 hour"}]}
{"question": "Which minimum Python version does the official SDK need?", "relevant_doc_ids": ["fleet-sdk-versions"], "expected_answer": "Python 3.10 or newer.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "fleet-sdk-versions", "span": "Python 3.10 or newer"}]}
{"question": "How often does the free test environment wipe its data?", "relevant_doc_ids": ["fleet-sandbox"], "expected_answer": "Every Sunday at 00:00 UTC.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "fleet-sandbox", "span": "resets its data every Sunday at 00:00 UTC"}]}
{"question": "What API request limit applies to a Pro customer in production versus in the sandbox?", "relevant_doc_ids": ["fleet-api-rate-limits", "fleet-sandbox"], "expected_answer": "600 requests per minute in production on Pro, and 120 requests per minute in the sandbox.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "fleet-api-rate-limits", "span": "600 on Pro"}, {"doc_id": "fleet-sandbox", "span": "API limits in the sandbox are 120 requests per minute regardless of plan"}]}
{"question": "A Starter customer wants to test webhooks, can they?", "relevant_doc_ids": ["fleet-webhooks", "fleet-sandbox"], "expected_answer": "Not in production because webhooks are Pro and Enterprise only, but webhooks work in the sandbox for testing.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "fleet-webhooks", "span": "Webhooks are available on Pro and Enterprise plans only"}, {"doc_id": "fleet-sandbox", "span": "Webhooks work in the sandbox for testing"}]}
{"question": "Does the fleet API support GraphQL queries?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How flat must the warehouse floor be?", "relevant_doc_ids": ["site-survey"], "expected_answer": "Flat within 5 millimetres over 2 metres.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "site-survey", "span": "flat within 5 millimetres over 2 metres"}]}
{"question": "How long can robots keep working if the wireless network drops?", "relevant_doc_ids": ["network-requirements"], "expected_answer": "Up to 30 minutes using cached tasks.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "network-requirements", "span": "no connection for up to 30 minutes"}]}
{"question": "How much does a trial deployment cost and how long does it last?", "relevant_doc_ids": ["pilot-program"], "expected_answer": "A fixed fee of 18,000 euros for up to 5 robots over 8 weeks.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "pilot-program", "span": "fixed fee of 18,000 euros"}]}
{"question": "How much on-site technician time is provided after a new site starts operating?", "relevant_doc_ids": ["go-live-checklist"], "expected_answer": "At least 4 hours every working day for the first two weeks.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "go-live-checklist", "span": "at least 4 hours every working day"}]}
{"question": "For how long can a customer still export reports after the contract ends?", "relevant_doc_ids": ["decommissioning"], "expected_answer": "60 days in read-only mode.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "decommissioning", "span": "read-only for 60 days"}]}
{"question": "A prospect wants a pilot, what visit comes first and how long until they get its report?", "relevant_doc_ids": ["pilot-program", "site-survey"], "expected_answer": "A site survey of one to two days comes first, and the report is delivered within 7 business days; the pilot includes the survey.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "pilot-program", "span": "The pilot includes the site survey"}, {"doc_id": "site-survey", "span": "delivered within 7 business days"}]}
{"question": "Do robots work in cold storage below minus 20 degrees?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "Who must approve a purchase above 100,000 euros?", "relevant_doc_ids": ["purchasing-thresholds"], "expected_answer": "The chief executive.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "purchasing-thresholds", "span": "above 100,000 euros from the chief executive"}]}
{"question": "How do we check a supplier's request to change their bank account?", "relevant_doc_ids": ["vendor-onboarding"], "expected_answer": "Changes are only accepted in writing and verified by phone with a known contact at the vendor.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "vendor-onboarding", "span": "verified by phone with a known contact at the vendor"}]}
{"question": "How large are the discounts negotiated with preferred suppliers?", "relevant_doc_ids": ["preferred-vendors"], "expected_answer": "Between 8 and 20 percent.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "preferred-vendors", "span": "between 8 and 20 percent"}]}
{"question": "How far ahead does procurement warn the contract owner before a term ends?", "relevant_doc_ids": ["contract-renewal"], "expected_answer": "90 days before the end of the term.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "contract-renewal", "span": "90 days before the end of the term"}]}
{"question": "How long does an approver have before a purchase request goes to their delegate?", "relevant_doc_ids": ["po-approval"], "expected_answer": "3 business days.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "po-approval", "span": "3 business days to respond"}]}
{"question": "For a 30,000 euro purchase, how many quotes are needed and who finally approves the purchase order?", "relevant_doc_ids": ["purchasing-thresholds", "po-approval"], "expected_answer": "Three written quotes, and the finance director gives the final approval.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "purchasing-thresholds", "span": "need three written quotes and approval from the finance director"}, {"doc_id": "po-approval", "span": "finally, above 25,000 euros, to the finance director"}]}
{"question": "Can employees buy software subscriptions with their personal credit card and claim them back?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "By what date do departments submit their budget requests?", "relevant_doc_ids": ["budget-cycle"], "expected_answer": "By 15 October.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "budget-cycle", "span": "by 15 October"}]}
{"question": "How long is an approved hiring slot valid before it lapses?", "relevant_doc_ids": ["headcount-planning"], "expected_answer": "6 months.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "headcount-planning", "span": "valid for 6 months"}]}
{"question": "From which date is subscription revenue counted?", "relevant_doc_ids": ["revenue-recognition"], "expected_answer": "From the go-live date, not the signing date.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "revenue-recognition", "span": "starting on the go-live date, not the signing date"}]}
{"question": "Within how many working days are the books closed after a quarter ends?", "relevant_doc_ids": ["quarter-close"], "expected_answer": "Within 8 working days.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "quarter-close", "span": "within 8 working days"}]}
{"question": "In which months do the outside auditors review the accounts?", "relevant_doc_ids": ["audit-prep"], "expected_answer": "February and March.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "audit-prep", "span": "every year in February and March"}]}
{"question": "How is the pilot fee treated for revenue, and how much of it can be credited against a later subscription?", "relevant_doc_ids": ["revenue-recognition", "pilot-program"], "expected_answer": "It is recognised over the pilot period and credited in full against the first year's subscription if the customer signs within 30 days.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "revenue-recognition", "span": "Pilot fees are recognised over the pilot period"}, {"doc_id": "pilot-program", "span": "credited in full against the first year's subscription"}]}
{"question": "When is the final budget approved, and how long does an approved new role stay valid?", "relevant_doc_ids": ["budget-cycle", "headcount-planning"], "expected_answer": "The budget is approved in the first week of December and an approved role is valid for 6 months.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "budget-cycle", "span": "in the first week of December"}, {"doc_id": "headcount-planning", "span": "valid for 6 months"}]}
{"question": "What is the travel budget for the legal department?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "What is the term of the standard mutual NDA?", "relevant_doc_ids": ["nda-policy"], "expected_answer": "3 years.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "nda-policy", "span": "term of 3 years"}]}
{"question": "May we ship GPL licensed code inside the robot software?", "relevant_doc_ids": ["open-source-policy"], "expected_answer": "No, GPL and AGPL libraries may not be included in shipped robot software.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "open-source-policy", "span": "may not be included in shipped robot software"}]}
{"question": "How long does an export licence typically take?", "relevant_doc_ids": ["export-controls"], "expected_answer": "Typically 4 to 8 weeks.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "export-controls", "span": "typically takes 4 to 8 weeks"}]}
{"question": "Can a customer print our logo in their own brochure?", "relevant_doc_ids": ["trademark-usage"], "expected_answer": "Only with written permission from the marketing director.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "trademark-usage", "span": "without written permission from the marketing director"}]}
{"question": "How soon does compliance acknowledge a misconduct report?", "relevant_doc_ids": ["whistleblower"], "expected_answer": "Within 7 days.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "whistleblower", "span": "acknowledges every report within 7 days"}]}
{"question": "Can I add a library licensed under the AGPL that has had no release in three years?", "relevant_doc_ids": ["open-source-policy", "dependency-policy"], "expected_answer": "No, AGPL libraries are not allowed in the shipped platform, and packages with no release in 24 months are not accepted for new work.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "open-source-policy", "span": "Libraries under the GPL and AGPL licences may not be included"}, {"doc_id": "dependency-policy", "span": "Packages with no release in the last 24 months are not accepted for new work"}]}
{"question": "Is there a bonus for filing a patent?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "What is the accent colour of the Halden brand?", "relevant_doc_ids": ["brand-guidelines"], "expected_answer": "Orange.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "brand-guidelines", "span": "orange as the accent"}]}
{"question": "Who is allowed to talk to reporters on behalf of the company?", "relevant_doc_ids": ["press-policy"], "expected_answer": "Only the head of communications and the chief executive.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "press-policy", "span": "Only the head of communications and the chief executive may speak to journalists"}]}
{"question": "How many events does marketing aim to sponsor each year at most?", "relevant_doc_ids": ["conference-sponsorship"], "expected_answer": "No more than 8.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "conference-sponsorship", "span": "no more than 8 sponsored events per year"}]}
{"question": "How long does a customer's consent to a success story stay valid?", "relevant_doc_ids": ["case-study-process"], "expected_answer": "24 months.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "case-study-process", "span": "valid for 24 months"}]}
{"question": "Who publishes on the official company social channels?", "relevant_doc_ids": ["social-media-policy"], "expected_answer": "Only the communications team.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "social-media-policy", "span": "Only the communications team publishes on the official company channels"}]}
{"question": "How much clear space does the logo need, and who approves a customer using it?", "relevant_doc_ids": ["brand-guidelines", "trademark-usage"], "expected_answer": "A clear space equal to the height of the letter H on all sides, and written permission from the marketing director.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "brand-guidelines", "span": "minimum clear space equal to the height of the letter H"}, {"doc_id": "trademark-usage", "span": "written permission from the marketing director"}]}
{"question": "What is the budget for the annual sales kickoff?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How many desks does the Frankfurt office have?", "relevant_doc_ids": ["frankfurt-office"], "expected_answer": "220 desks.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "frankfurt-office", "span": "220 desks"}]}
{"question": "Does the Austin site have a prototyping lab?", "relevant_doc_ids": ["austin-office"], "expected_answer": "No, there is no prototyping lab at the Austin site.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "austin-office", "span": "there is no prototyping lab at this site"}]}
{"question": "What identification must a guest bring to the Pune security desk?", "relevant_doc_ids": ["pune-office"], "expected_answer": "A government photo ID.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "pune-office", "span": "carry a government photo ID"}]}
{"question": "May I take photos on the test warehouse floor?", "relevant_doc_ids": ["visitor-policy"], "expected_answer": "Not without written approval from the site manager.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "visitor-policy", "span": "Photography is not allowed in prototyping areas or on warehouse test floors without written approval from the site manager"}]}
{"question": "What is the monthly parking fee in Frankfurt?", "relevant_doc_ids": ["parking"], "expected_answer": "90 euros per month.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "parking", "span": "the monthly fee is 90 euros"}]}
{"question": "Can an on-call engineer enter the Frankfurt office at night, and how are parking places there handed out?", "relevant_doc_ids": ["frankfurt-office", "parking"], "expected_answer": "Yes, badge access is available around the clock for on-call engineers, and parking is allocated by lottery every quarter.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "frankfurt-office", "span": "badge access is available around the clock for engineers on call"}, {"doc_id": "parking", "span": "allocated by lottery every quarter"}]}
{"question": "Is there a gym at the Pune office?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How often must an internal wiki page be reviewed?", "relevant_doc_ids": ["wiki-guidelines"], "expected_answer": "At least once a year.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "wiki-guidelines", "span": "reviewed at least once a year"}]}
{"question": "Am I supposed to reply to chat messages on my days off?", "relevant_doc_ids": ["slack-etiquette"], "expected_answer": "No, employees are not expected to answer outside their working hours.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "slack-etiquette", "span": "not expected to answer outside their working hours"}]}
{"question": "Which weekday has no meetings for engineering teams?", "relevant_doc_ids": ["meeting-policy"], "expected_answer": "Wednesday.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "meeting-policy", "span": "Wednesdays are meeting-free for engineering teams"}]}
{"question": "How long can a single VPN session last?", "relevant_doc_ids": ["vpn-access"], "expected_answer": "12 hours.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "vpn-access", "span": "disconnect after 12 hours"}]}
{"question": "What is the response target when I am locked out of my account?", "relevant_doc_ids": ["it-helpdesk"], "expected_answer": "30 minutes.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "it-helpdesk", "span": "response target of 30 minutes"}]}
{"question": "What do I need to connect to the VPN, and are SMS codes accepted for it?", "relevant_doc_ids": ["vpn-access", "mfa-policy"], "expected_answer": "A hardware security key and a company-managed laptop, and text message codes are not accepted.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "vpn-access", "span": "needs a hardware security key and a company-managed laptop"}, {"doc_id": "mfa-policy", "span": "text message codes are not accepted for any system"}]}
{"question": "Which wiki software does the company use?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How many approvals does a billing change need?", "relevant_doc_ids": ["code-review-standards"], "expected_answer": "Two approvals.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "code-review-standards", "span": "two approvals for changes to billing or authentication"}]}
{"question": "What coverage level blocks a pull request from being merged?", "relevant_doc_ids": ["testing-standards"], "expected_answer": "Coverage of changed lines below 80 percent.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "testing-standards", "span": "falls below 80 percent"}]}
{"question": "How long may a working branch stay open?", "relevant_doc_ids": ["branching-strategy"], "expected_answer": "No longer than 2 days.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "branching-strategy", "span": "no longer than 2 days"}]}
{"question": "How quickly must a critical vulnerability in a library be patched?", "relevant_doc_ids": ["dependency-policy"], "expected_answer": "Within 7 days for high or critical ratings.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "dependency-policy", "span": "must be applied within 7 days"}]}
{"question": "What share of each sprint is reserved for technical debt?", "relevant_doc_ids": ["tech-debt-policy"], "expected_answer": "20 percent.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "tech-debt-policy", "span": "20 percent of every sprint"}]}
{"question": "What reviews and sign-offs does a change to billing behaviour need before it is switched on for customers?", "relevant_doc_ids": ["code-review-standards", "feature-flag-policy"], "expected_answer": "Two code review approvals, and sign-off from the finance engineering lead before the feature flag is turned on.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "code-review-standards", "span": "two approvals for changes to billing or authentication"}, {"doc_id": "feature-flag-policy", "span": "need sign-off from the finance engineering lead"}]}
{"question": "What is the policy on pair programming?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How long does standard customer onboarding take?", "relevant_doc_ids": ["onboarding-customer"], "expected_answer": "6 weeks from kickoff to go-live.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "onboarding-customer", "span": "takes 6 weeks from kickoff to go-live"}]}
{"question": "How often do Pro customers get a business review?", "relevant_doc_ids": ["qbr-process"], "expected_answer": "Twice a year.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "qbr-process", "span": "Pro customers receive a review twice a year"}]}
{"question": "What fall in robot utilisation marks an account as at risk?", "relevant_doc_ids": ["churn-risk-playbook"], "expected_answer": "A drop of more than 30 percent over two months.", "category": "disambiguation", "answerable": true, "evidence": [{"doc_id": "churn-risk-playbook", "span": "falls by more than 30 percent over two months"}]}
{"question": "At what point before the end of the contract does the customer receive a renewal quote?", "relevant_doc_ids": ["renewal-process"], "expected_answer": "At 90 days before the end.", "category": "paraphrase", "answerable": true, "evidence": [{"doc_id": "renewal-process", "span": "A renewal quote is sent at 90 days"}]}
{"question": "What is the target net promoter score?", "relevant_doc_ids": ["nps-survey"], "expected_answer": "45 or higher.", "category": "direct", "answerable": true, "evidence": [{"doc_id": "nps-survey", "span": "target score is 45 or higher"}]}
{"question": "After the standard onboarding, who signs the go-live checklist and how much on-site support follows?", "relevant_doc_ids": ["onboarding-customer", "go-live-checklist"], "expected_answer": "A supervisor and the Halden deployment lead sign it, and a technician is on site at least 4 hours every working day for two weeks.", "category": "multihop", "answerable": true, "evidence": [{"doc_id": "onboarding-customer", "span": "takes 6 weeks from kickoff to go-live"}, {"doc_id": "go-live-checklist", "span": "A supervisor and the Halden deployment lead both sign the checklist"}]}
{"question": "What was the net promoter score last quarter?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": []}
{"question": "How much notice must employees give before starting parental leave?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Can Pune employees take leave in half-day units?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Is the home office grant taxable for employees based in Singapore?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which monitor brands are approved under the monitor stipend?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Is there an extra on-call payment for weeks that include a public holiday?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which travel agency books international flights for employees?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What is the onsite meal allowance in rupees for deployments in India?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How much does parking cost at the Singapore office?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which brand of hardware security key does the company issue?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How many failed login attempts lock a company account?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which VPN client software do employees install?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which encryption algorithm is used for full-disk encryption on company laptops?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Is there a free trial of the Pro plan?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What does the Starter plan cost with quarterly billing?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What is the first response time for non-critical Enterprise tickets?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What are the support hours for customers in Latin America?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What is the operating temperature range of the H200 robot?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How much does an H400 robot weigh?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What battery chemistry do the robots use?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How much does a charging station cost?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which company manufactures the emergency stop buttons?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Does the company pay for employees' safety shoes?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "From which IP addresses are webhook deliveries sent?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How many API tokens can one customer create?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How long are security camera recordings kept?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What is the deadline for answering a GDPR request for a copy of personal data?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Is there a bonus for referring a new customer?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Can employees appeal their performance review rating?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What salary increase comes with a promotion?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Is there a cafeteria at the Pune office?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What is the street address of the Frankfurt office?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Who is the site manager of the Austin office?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Can visitors use a guest Wi-Fi network?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How many hotfixes were shipped last quarter?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "How long are previous container images kept for rollbacks?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which bank issues the company's corporate cards?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "In which currency are purchase orders raised for the Austin office?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "Which conferences did the company sponsor last year?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What is the deductible on the company travel insurance?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}
{"question": "What is the purchase price of an H200 robot?", "relevant_doc_ids": [], "expected_answer": null, "category": "unanswerable", "answerable": false, "evidence": [], "near_miss": true}