File size: 1,712 Bytes
d92822d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | """Illustrative support-app logic; reads saved answers and never takes actions."""
import argparse
import json
from pathlib import Path
def decide(answers):
return {"route": answers["department"]["choice"],
"refund_review": answers["refund"]["noul"] >= .5,
"urgent_queue": answers["severity"]["score"] >= 1.5}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("comparison_directory")
args = parser.parse_args()
root = Path(args.comparison_directory)
summary = json.loads((root/"summary.json").read_text(encoding="utf-8"))
if summary["status"] != "complete":
raise ValueError("Use a completed comparison; do not hide provider failures")
combined, fingerprints = {}, {}
for provider in summary["providers"]:
records = [json.loads(line) for line in (root/f"{provider}-responses.jsonl").read_text(encoding="utf-8").splitlines()]
for record in records:
key = record["case_id"]
if key in fingerprints and fingerprints[key] != record["request_sha256"]:
raise ValueError("Cannot compare decisions on different requests")
fingerprints[key] = record["request_sha256"]
combined.setdefault(key, {})[provider] = decide(record["response"]["answers"])
for key, decisions in combined.items():
if set(decisions) != set(summary["providers"]):
raise ValueError("Incomplete provider coverage")
distinct = {json.dumps(d, sort_keys=True) for d in decisions.values()}
print(json.dumps({"case_id": key, "decisions": decisions, "providers_disagree": len(distinct)>1}))
if __name__ == "__main__":
main()
|