[ { "id": 1, "client": "novamart", "question": "What happens when a product runs out of stock?", "error": "timeout" }, { "id": 2, "client": "novamart", "question": "How do I onboard a new merchant?", "answer": "The information doesn't provide details on merchant onboarding. I can only provide a general statement, as the context only discusses supplier setup.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.724, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 3, "client": "novamart", "question": "What is an availability scan and when does it trigger?", "answer": "I cannot confirm information about what an availability scan is.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.876, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 4, "client": "novamart", "question": "How do I update pricing for an item in the item registry?", "answer": "To update pricing for an item in the item registry, you must submit a price update (pricing sync) through the pricing portal. Changes greater than 15% of the current price require approval, while smaller changes can be approved automatically. Approved changes will go live at the next scheduled sync window, which runs every four hours.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.839, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 5, "client": "novamart", "question": "What is a capability switch and how do I enable one?", "answer": "A capability switch is also referred to as a feature flag. To enable one in production, you will need to get sign-off from both the product and engineering lead.", "overall": "PASS", "flagged": false, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.799, "faithfulness": 0.889, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": true, "chain_terminology": true } }, { "id": 6, "client": "shelfwise", "question": "What triggers an out-of-stock alert?", "answer": "A product's on-hand quantity dropping below its configured reorder point triggers an out-of-stock alert.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.677, "faithfulness": 0.026, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 7, "client": "shelfwise", "question": "How does planogram compliance work?", "answer": "I couldn't find any information in the provided context about planogram compliance. It seems that the context only covers compliance reporting, store configuration, and supplier setup.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.791, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 8, "client": "shelfwise", "question": "How do I add a new store to the system?", "answer": "Unfortunately, the provided context does not contain the information on how to add a new store to the system.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.849, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 9, "client": "shelfwise", "question": "How do I generate a compliance report for a store?", "answer": "To generate a compliance report for a store, access must be granted to a user with the Audit role or higher.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.759, "faithfulness": 0.038, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 10, "client": "shelfwise", "question": "What happens if a feature toggle is disabled mid-deployment?", "answer": "The context does not specify what happens if a feature toggle is disabled mid-deployment.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.887, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 11, "client": "clinixone", "question": "What is prior authorization and when is it required?", "answer": "Prior authorization, also known as formulary pre-approval, is a requirement by a payer that a prescriber obtain approval before a specific drug is dispensed and covered. It is needed when a payer wants the prescriber to submit clinical justification for the specific drug.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.735, "faithfulness": 0.021, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 12, "client": "clinixone", "question": "What are contraindicated drug combinations and how are they flagged?", "answer": "Contraindicated drug combinations occur when one drug affects the activity of another when both are administered together. \n\nThese combinations are flagged by alerting prescribers at point-of-care when a contraindicated combination is entered. The severity classification of interactions follows the clinical pharmacology guidelines maintained in the formulary interaction database.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.802, "faithfulness": 0.003, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 13, "client": "clinixone", "question": "How should adverse events be reported?", "answer": "Adverse events must be reported to the regulatory authority within 15 days for serious unexpected events and 90 days for expected events.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.676, "faithfulness": 0.117, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 14, "client": "clinixone", "question": "What is a safety signal and how is it escalated?", "answer": "A safety signal (pharmacovigilance alert) is a pattern of adverse events that suggests a previously unknown or incompletely documented causal relationship between a drug and an outcome. Adverse events must be reported to the regulatory authority within 15 days for serious unexpected events and 90 days for expected events.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.644, "faithfulness": 0.001, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 15, "client": "clinixone", "question": "Which drugs are contraindicated in patients with bleeding disorders?", "answer": "The context does not mention any contraindication for patients with bleeding disorders.", "overall": "PASS", "flagged": false, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.665, "faithfulness": 1.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": true, "chain_terminology": true } }, { "id": 16, "client": "clinixone", "question": "What adverse events are associated with drugs for allergy relief?", "answer": "There is no information about drugs specifically for allergy relief in the context provided.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.741, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 17, "client": "clinixone", "question": "What drugs cause fatigue as a known adverse event?", "answer": "Amoxstatin62, Amoxzole72, and Mecozole85 cause fatigue as a known adverse event.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.715, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 18, "client": "pharmalink", "question": "What is formulary pre-approval?", "answer": "Formulary pre-approval is a prior authorization requirement by a payer that a prescriber obtain approval before a specific drug is dispensed and covered.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.869, "faithfulness": 0.003, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 19, "client": "pharmalink", "question": "What is a pharmacovigilance alert and when does it fire?", "answer": "A pharmacovigilance alert, also known as a safety signal, is a pattern of adverse events that suggests a previously unknown or incompletely documented causal relationship between a drug and an outcome. \n\nIt suggests a possible causal relationship but does not confirm one.", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.811, "faithfulness": 0.005, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } }, { "id": 20, "client": "pharmalink", "question": "Which drugs have a prescriber warning about avoiding alcohol?", "answer": "There's no information in the provided context regarding a prescriber warning about avoiding alcohol for any of the mentioned drugs (Zylozole96, Zylozepam79, Zylozole29).", "overall": "FAIL", "flagged": true, "scores": { "pii_leakage": 1.0, "token_budget": 1.0, "answer_relevancy": 0.792, "faithfulness": 0.0, "chain_terminology": 1.0 }, "passed": { "pii_leakage": true, "token_budget": true, "answer_relevancy": true, "faithfulness": false, "chain_terminology": true } } ]