| [ |
| { |
| "id": "MARTA-A-001", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Airport to Five Points", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-002", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Airport to Midtown", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Midtown", |
| "station_id": "MARTA-MT" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.76, |
| "estimated_minutes": 21.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-003", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Buckhead to Airport", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 15.58, |
| "estimated_minutes": 31.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-004", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Decatur to Five Points", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Decatur", |
| "station_id": "MARTA-DC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.01, |
| "estimated_minutes": 11.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-005", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "North Springs to Airport", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs", |
| "station_id": "MARTA-NS" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-006", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Doraville to Airport", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Doraville", |
| "station_id": "MARTA-DO" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 21.51, |
| "estimated_minutes": 43.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-007", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Lindbergh Center to Five Points", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Lindbergh Center", |
| "station_id": "MARTA-LC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 5.1, |
| "estimated_minutes": 10.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-008", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Indian Creek to Five Points", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-IC", |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 9.86, |
| "estimated_minutes": 19.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-009", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Bankhead to Five Points", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BK", |
| "MARTA-AS", |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 2.78, |
| "estimated_minutes": 5.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-010", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "memorizable", |
| "title": "Edgewood/Candler Park to Airport", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park", |
| "station_id": "MARTA-EC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "green", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 11.98, |
| "estimated_minutes": 28.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-011", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Edgewood/Candler Park to Bankhead", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park", |
| "station_id": "MARTA-EC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS", |
| "MARTA-BK" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 5.97, |
| "estimated_minutes": 11.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-012", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Edgewood/Candler Park to Sandy Springs", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park", |
| "station_id": "MARTA-EC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Sandy Springs", |
| "station_id": "MARTA-SS" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW", |
| "MARTA-SS" |
| ], |
| "line_sequence": [ |
| "green", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 16.09, |
| "estimated_minutes": 37.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-013", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Bankhead to North Springs", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "North Springs", |
| "station_id": "MARTA-NS" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BK", |
| "MARTA-AS", |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW", |
| "MARTA-SS", |
| "MARTA-NS" |
| ], |
| "line_sequence": [ |
| "green", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 16.63, |
| "estimated_minutes": 38.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-014", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Lenox to Bankhead", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Lenox", |
| "station_id": "MARTA-LX" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS", |
| "MARTA-BK" |
| ], |
| "line_sequence": [ |
| "gold", |
| "green" |
| ], |
| "transfers": 1, |
| "distance_miles": 9.52, |
| "estimated_minutes": 24.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-015", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Indian Creek to Sandy Springs", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Sandy Springs", |
| "station_id": "MARTA-SS" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-IC", |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW", |
| "MARTA-SS" |
| ], |
| "line_sequence": [ |
| "blue", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 22.76, |
| "estimated_minutes": 50.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-016", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Bankhead to Chamblee", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Chamblee", |
| "station_id": "MARTA-CH" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BK", |
| "MARTA-AS", |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-LX", |
| "MARTA-BO", |
| "MARTA-CH" |
| ], |
| "line_sequence": [ |
| "green", |
| "gold" |
| ], |
| "transfers": 1, |
| "distance_miles": 13.74, |
| "estimated_minutes": 32.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-017", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Edgewood/Candler Park to Ashby", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park", |
| "station_id": "MARTA-EC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Ashby", |
| "station_id": "MARTA-AS" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 4.68, |
| "estimated_minutes": 9.3 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-018", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Vine City to Indian Creek", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Vine City", |
| "station_id": "MARTA-VC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.59, |
| "estimated_minutes": 21.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-019", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "OMNI/Dome/GWCC/Philips Arena/CNN Center to Dunwoody", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "OMNI/Dome/GWCC/Philips Arena/CNN Center", |
| "station_id": "MARTA-OM" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Dunwoody", |
| "station_id": "MARTA-DW" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW" |
| ], |
| "line_sequence": [ |
| "green", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 12.4, |
| "estimated_minutes": 29.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-A-020", |
| "system": "marta", |
| "category": "A", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "route_type": "novel", |
| "title": "Edgewood/Candler Park to Doraville", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park", |
| "station_id": "MARTA-EC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Doraville", |
| "station_id": "MARTA-DO" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-LX", |
| "MARTA-BO", |
| "MARTA-CH", |
| "MARTA-DO" |
| ], |
| "line_sequence": [ |
| "green", |
| "gold" |
| ], |
| "transfers": 1, |
| "distance_miles": 15.91, |
| "estimated_minutes": 36.8 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-001", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 adult", |
| "title": "Fare: 1 adult", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x1", |
| "amount": 2.5, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 2.5, |
| "discounts": [], |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-002", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "2 adults + 1 child", |
| "title": "Fare: 2 adults + 1 child", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x2", |
| "amount": 5.0, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 5.0, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x1", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 2, |
| "children": 1, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 1 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 2, |
| "unit_fare": 2.5, |
| "subtotal": 5.0, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-003", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 adult + 3 children", |
| "title": "Fare: 1 adult + 3 children", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 3 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x1", |
| "amount": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "label": "Child (fare required) x1", |
| "amount": 2.5, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 5.0, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x2", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 3, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 2 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "child", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-004", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "2 seniors", |
| "title": "Fare: 2 seniors", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "seniors": 2 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Senior x2", |
| "amount": 2.5, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 2.5, |
| "discounts": [], |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 0, |
| "children": 0, |
| "seniors": 2, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "senior", |
| "count": 2, |
| "unit_fare": 1.25, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-005", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 adult + 1 senior + 1 disabled", |
| "title": "Fare: 1 adult + 1 senior + 1 disabled", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1, |
| "disabled": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x1", |
| "amount": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "label": "Senior x1", |
| "amount": 1.25, |
| "currency": "USD" |
| }, |
| { |
| "label": "Disabled x1", |
| "amount": 1.25, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 5.0, |
| "discounts": [], |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 0, |
| "seniors": 1, |
| "disabled": 1, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "senior", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "disabled", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-006", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 adult + 1 child + 1 senior", |
| "title": "Fare: 1 adult + 1 child + 1 senior", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 1, |
| "seniors": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x1", |
| "amount": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "label": "Senior x1", |
| "amount": 1.25, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 3.75, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x1", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 3.75, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 1, |
| "seniors": 1, |
| "disabled": 0, |
| "free_riders": 1 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "senior", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-007", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "3 adults", |
| "title": "Fare: 3 adults", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 3 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x3", |
| "amount": 7.5, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 7.5, |
| "discounts": [], |
| "total": 7.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 3, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 3, |
| "unit_fare": 2.5, |
| "subtotal": 7.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-008", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 disabled", |
| "title": "Fare: 1 disabled", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "disabled": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Disabled x1", |
| "amount": 1.25, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 1.25, |
| "discounts": [], |
| "total": 1.25, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 0, |
| "children": 0, |
| "seniors": 0, |
| "disabled": 1, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "disabled", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-009", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "2 adults + 3 children", |
| "title": "Fare: 2 adults + 3 children", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 3 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x2", |
| "amount": 5.0, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 5.0, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x3", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 2, |
| "children": 3, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 3 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 2, |
| "unit_fare": 2.5, |
| "subtotal": 5.0, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-010", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "0 adults + 2 children", |
| "title": "Fare: 0 adults + 2 children", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "children": 2 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Child (fare required) x2", |
| "amount": 5.0, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 5.0, |
| "discounts": [], |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 0, |
| "children": 2, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 0 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "child", |
| "count": 2, |
| "unit_fare": 2.5, |
| "subtotal": 5.0, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-011", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "2 adults + 2 children + 1 senior + 1 disabled", |
| "title": "Fare: 2 adults + 2 children + 1 senior + 1 disabled", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 2, |
| "seniors": 1, |
| "disabled": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x2", |
| "amount": 5.0, |
| "currency": "USD" |
| }, |
| { |
| "label": "Senior x1", |
| "amount": 1.25, |
| "currency": "USD" |
| }, |
| { |
| "label": "Disabled x1", |
| "amount": 1.25, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 7.5, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x2", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 7.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 2, |
| "children": 2, |
| "seniors": 1, |
| "disabled": 1, |
| "free_riders": 2 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 2, |
| "unit_fare": 2.5, |
| "subtotal": 5.0, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "senior", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "disabled", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-012", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 adult + 2 children (max free hit)", |
| "title": "Fare: 1 adult + 2 children (max free hit)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 2 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x1", |
| "amount": 2.5, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 2.5, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x2", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 2, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 2 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-013", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 adult + 4 children (2 free 2 pay)", |
| "title": "Fare: 1 adult + 4 children (2 free 2 pay)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 4 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x1", |
| "amount": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "label": "Child (fare required) x2", |
| "amount": 5.0, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 7.5, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x2", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 7.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 1, |
| "children": 4, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 2 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 1, |
| "unit_fare": 2.5, |
| "subtotal": 2.5, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "child", |
| "count": 2, |
| "unit_fare": 2.5, |
| "subtotal": 5.0, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-014", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "2 adults + 4 children", |
| "title": "Fare: 2 adults + 4 children", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 4 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Adult x2", |
| "amount": 5.0, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 5.0, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x4", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 2, |
| "children": 4, |
| "seniors": 0, |
| "disabled": 0, |
| "free_riders": 4 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "adult", |
| "count": 2, |
| "unit_fare": 2.5, |
| "subtotal": 5.0, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-B-015", |
| "system": "marta", |
| "category": "B", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "passenger_composition": "1 senior + 1 disabled + 2 children", |
| "title": "Fare: 1 senior + 1 disabled + 2 children", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "children": 2, |
| "seniors": 1, |
| "disabled": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "items": [ |
| { |
| "label": "Senior x1", |
| "amount": 1.25, |
| "currency": "USD" |
| }, |
| { |
| "label": "Disabled x1", |
| "amount": 1.25, |
| "currency": "USD" |
| } |
| ], |
| "subtotal": 2.5, |
| "discounts": [ |
| { |
| "label": "Child (under 5, free) x2", |
| "amount": 0.0, |
| "currency": "USD" |
| } |
| ], |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok", |
| "expected_fare_breakdown": { |
| "passenger_summary": { |
| "adults": 0, |
| "children": 2, |
| "seniors": 1, |
| "disabled": 1, |
| "free_riders": 2 |
| }, |
| "line_items": [ |
| { |
| "rider_type": "senior", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| }, |
| { |
| "rider_type": "disabled", |
| "count": 1, |
| "unit_fare": 1.25, |
| "subtotal": 1.25, |
| "currency": "USD" |
| } |
| ] |
| } |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "fare_breakdown_correct": 5, |
| "passenger_summary_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-001", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "sc_five-points", |
| "title": "Airport to Indian Creek (station_closure)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-five-points", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "critical", |
| "message": "Five Points station closed due to emergency structural inspection. Trains will skip this station. Use Garnett or Peachtree Center as alternatives.", |
| "alternative": "Use Garnett (southbound) or Peachtree Center (northbound)", |
| "eta_resolution": "4-6 hours" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-five-points", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "critical", |
| "message": "Five Points station closed due to emergency structural inspection. Trains will skip this station. Use Garnett or Peachtree Center as alternatives.", |
| "alternative": "Use Garnett (southbound) or Peachtree Center (northbound)", |
| "eta_resolution": "4-6 hours" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "line_sequence": [ |
| "gold", |
| "blue" |
| ], |
| "transfers": 1, |
| "distance_miles": 18.65, |
| "estimated_minutes": 42.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "line_sequence": [ |
| "gold", |
| "blue" |
| ], |
| "transfers": 1, |
| "distance_miles": 18.65, |
| "estimated_minutes": 42.2 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "skip", |
| "expected_restrictions": [ |
| { |
| "station": "MARTA-FP", |
| "restriction": "skip" |
| } |
| ], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "critical", |
| "advisory_must_mention": [ |
| "five points", |
| "closed", |
| "structural" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-002", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "disruption_type": "sc_midtown", |
| "title": "Buckhead to Five Points (station_closure)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.79, |
| "estimated_minutes": 13.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.79, |
| "estimated_minutes": 13.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.79, |
| "estimated_minutes": 13.6 |
| }, |
| "restriction_type": "skip", |
| "expected_restrictions": [ |
| { |
| "station": "MARTA-MT", |
| "restriction": "skip" |
| } |
| ], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "midtown", |
| "closed", |
| "water main" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-003", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "sc_airport", |
| "title": "Five Points to Airport (station_closure)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-airport", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "critical", |
| "message": "Airport station closed due to security incident at airport terminal. No train service to Airport. Use College Park station and airport shuttle as alternative.", |
| "alternative": "Use College Park station and airport shuttle service", |
| "eta_resolution": "unknown" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-airport", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "critical", |
| "message": "Airport station closed due to security incident at airport terminal. No train service to Airport. Use College Park station and airport shuttle as alternative.", |
| "alternative": "Use College Park station and airport shuttle service", |
| "eta_resolution": "unknown" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "skip", |
| "expected_restrictions": [ |
| { |
| "station": "MARTA-AP", |
| "restriction": "skip" |
| } |
| ], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "critical", |
| "advisory_must_mention": [ |
| "airport", |
| "closed", |
| "security" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-004", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "disruption_type": "sc_lindbergh", |
| "title": "North Springs to Airport (station_closure)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs", |
| "station_id": "MARTA-NS" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-lindbergh", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "critical", |
| "message": "Lindbergh Center station closed due to suspicious package investigation. Red and Gold line trains will skip this station. Use Arts Center or Buckhead as alternatives.", |
| "alternative": "Use Arts Center (southbound) or Buckhead (northbound/Red); Lenox (Gold)", |
| "eta_resolution": "1-3 hours" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-lindbergh", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "critical", |
| "message": "Lindbergh Center station closed due to suspicious package investigation. Red and Gold line trains will skip this station. Use Arts Center or Buckhead as alternatives.", |
| "alternative": "Use Arts Center (southbound) or Buckhead (northbound/Red); Lenox (Gold)", |
| "eta_resolution": "1-3 hours" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "restriction_type": "skip", |
| "expected_restrictions": [ |
| { |
| "station": "MARTA-LC", |
| "restriction": "skip" |
| } |
| ], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "critical", |
| "advisory_must_mention": [ |
| "lindbergh", |
| "closed", |
| "suspicious package" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-005", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "disruption_type": "sc_inman-park", |
| "title": "Edgewood/Candler Park to Five Points (station_closure)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park", |
| "station_id": "MARTA-EC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-inman-park", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Inman Park/Reynoldstown station closed for track defect repair. Blue and Green line trains will skip this station. Use King Memorial, East Lake, or Edgewood/Candler Park as alternatives.", |
| "alternative": "Use King Memorial (westbound) or East Lake (eastbound/Blue) or Edgewood/Candler Park (Green)", |
| "eta_resolution": "3-5 hours" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-inman-park", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Inman Park/Reynoldstown station closed for track defect repair. Blue and Green line trains will skip this station. Use King Memorial, East Lake, or Edgewood/Candler Park as alternatives.", |
| "alternative": "Use King Memorial (westbound) or East Lake (eastbound/Blue) or Edgewood/Candler Park (Green)", |
| "eta_resolution": "3-5 hours" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 3.19, |
| "estimated_minutes": 6.3 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 3.19, |
| "estimated_minutes": 6.3 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 3.19, |
| "estimated_minutes": 6.3 |
| }, |
| "restriction_type": "skip", |
| "expected_restrictions": [ |
| { |
| "station": "MARTA-IR", |
| "restriction": "skip" |
| } |
| ], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "inman park", |
| "closed", |
| "track defect" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-006", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "pm_red-south", |
| "title": "Five Points to Airport (planned_maintenance)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-red-south", |
| "line": "red", |
| "segment": [ |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Red Line: No service between Garnett and Airport this weekend due to track maintenance. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Garnett and Airport", |
| "eta_resolution": "Service resumes Tuesday 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-red-south", |
| "line": "red", |
| "segment": [ |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Red Line: No service between Garnett and Airport this weekend due to track maintenance. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Garnett and Airport", |
| "eta_resolution": "Service resumes Tuesday 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-GA", |
| "MARTA-WE" |
| ], |
| [ |
| "MARTA-WE", |
| "MARTA-OC" |
| ], |
| [ |
| "MARTA-OC", |
| "MARTA-LF" |
| ], |
| [ |
| "MARTA-LF", |
| "MARTA-EP" |
| ], |
| [ |
| "MARTA-EP", |
| "MARTA-CP" |
| ], |
| [ |
| "MARTA-CP", |
| "MARTA-AP" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "red line", |
| "garnett", |
| "airport", |
| "bus replacement", |
| "weekend" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-007", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "pm_blue-east", |
| "title": "Indian Creek to Five Points (planned_maintenance)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-blue-east", |
| "line": "blue", |
| "segment": [ |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Blue Line: No late-night service between East Lake and Indian Creek due to signal upgrade work. Last train departs East Lake at 10:00 PM.", |
| "alternative": "No replacement service; plan travel before 10:00 PM", |
| "eta_resolution": "Normal service resumes at 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-blue-east", |
| "line": "blue", |
| "segment": [ |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Blue Line: No late-night service between East Lake and Indian Creek due to signal upgrade work. Last train departs East Lake at 10:00 PM.", |
| "alternative": "No replacement service; plan travel before 10:00 PM", |
| "eta_resolution": "Normal service resumes at 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-IC", |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 9.86, |
| "estimated_minutes": 19.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-IC", |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 9.86, |
| "estimated_minutes": 19.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-EL", |
| "MARTA-DC" |
| ], |
| [ |
| "MARTA-DC", |
| "MARTA-AV" |
| ], |
| [ |
| "MARTA-AV", |
| "MARTA-KN" |
| ], |
| [ |
| "MARTA-KN", |
| "MARTA-IC" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "info", |
| "advisory_must_mention": [ |
| "blue line", |
| "east lake", |
| "indian creek", |
| "night", |
| "signal" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-008", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "pm_gold-north", |
| "title": "Doraville to Five Points (planned_maintenance)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Doraville", |
| "station_id": "MARTA-DO" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-gold-north", |
| "line": "gold", |
| "segment": [ |
| "MARTA-LX", |
| "MARTA-BO", |
| "MARTA-CH", |
| "MARTA-DO" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Gold Line: No service between Lenox and Doraville all day due to platform renovation. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Lenox and Doraville", |
| "eta_resolution": "Service resumes tomorrow 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-gold-north", |
| "line": "gold", |
| "segment": [ |
| "MARTA-LX", |
| "MARTA-BO", |
| "MARTA-CH", |
| "MARTA-DO" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Gold Line: No service between Lenox and Doraville all day due to platform renovation. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Lenox and Doraville", |
| "eta_resolution": "Service resumes tomorrow 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 12.72, |
| "estimated_minutes": 25.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 12.72, |
| "estimated_minutes": 25.5 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-LX", |
| "MARTA-BO" |
| ], |
| [ |
| "MARTA-BO", |
| "MARTA-CH" |
| ], |
| [ |
| "MARTA-CH", |
| "MARTA-DO" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "gold line", |
| "lenox", |
| "doraville", |
| "bus replacement" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-009", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "pm_red-north", |
| "title": "North Springs to Five Points (planned_maintenance)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs", |
| "station_id": "MARTA-NS" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-red-north", |
| "line": "red", |
| "segment": [ |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW", |
| "MARTA-SS", |
| "MARTA-NS" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Red Line: No service between Buckhead and North Springs this weekend due to rail replacement. Free shuttle service available between affected stations.", |
| "alternative": "Free shuttle service between Buckhead and North Springs", |
| "eta_resolution": "Service resumes Tuesday 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-red-north", |
| "line": "red", |
| "segment": [ |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW", |
| "MARTA-SS", |
| "MARTA-NS" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Red Line: No service between Buckhead and North Springs this weekend due to rail replacement. Free shuttle service available between affected stations.", |
| "alternative": "Free shuttle service between Buckhead and North Springs", |
| "eta_resolution": "Service resumes Tuesday 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 13.85, |
| "estimated_minutes": 27.8 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 13.85, |
| "estimated_minutes": 27.8 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-BH", |
| "MARTA-MC" |
| ], |
| [ |
| "MARTA-MC", |
| "MARTA-DW" |
| ], |
| [ |
| "MARTA-DW", |
| "MARTA-SS" |
| ], |
| [ |
| "MARTA-SS", |
| "MARTA-NS" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "red line", |
| "buckhead", |
| "north springs", |
| "shuttle", |
| "weekend" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-010", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "pm_blue-west", |
| "title": "Five Points to Bankhead (planned_maintenance)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-blue-west", |
| "line": "blue", |
| "segment": [ |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS", |
| "MARTA-BK" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Blue Line: No service between Five Points and Bankhead all day due to track geometry correction. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Five Points and Bankhead", |
| "eta_resolution": "Service resumes tomorrow 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-blue-west", |
| "line": "blue", |
| "segment": [ |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS", |
| "MARTA-BK" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Blue Line: No service between Five Points and Bankhead all day due to track geometry correction. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Five Points and Bankhead", |
| "eta_resolution": "Service resumes tomorrow 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS", |
| "MARTA-BK" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 2.78, |
| "estimated_minutes": 5.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS", |
| "MARTA-BK" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 2.78, |
| "estimated_minutes": 5.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-FP", |
| "MARTA-OM" |
| ], |
| [ |
| "MARTA-OM", |
| "MARTA-VC" |
| ], |
| [ |
| "MARTA-VC", |
| "MARTA-AS" |
| ], |
| [ |
| "MARTA-AS", |
| "MARTA-BK" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "blue line", |
| "five points", |
| "bankhead", |
| "bus replacement" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-011", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "disruption_type": "hw_approaching", |
| "title": "Airport to North Springs (hurricane_warning)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "North Springs", |
| "station_id": "MARTA-NS" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "hw-approaching", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "info", |
| "message": "Hurricane advisory: A hurricane is approaching the Atlanta metro area. All MARTA rail lines are currently operating normally. Passengers should monitor weather updates and plan travel accordingly.", |
| "alternative": null, |
| "eta_resolution": "Monitoring situation" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "hw-approaching", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "info", |
| "message": "Hurricane advisory: A hurricane is approaching the Atlanta metro area. All MARTA rail lines are currently operating normally. Passengers should monitor weather updates and plan travel accordingly.", |
| "alternative": null, |
| "eta_resolution": "Monitoring situation" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW", |
| "MARTA-SS", |
| "MARTA-NS" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-BH", |
| "MARTA-MC", |
| "MARTA-DW", |
| "MARTA-SS", |
| "MARTA-NS" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "post_disruption": { |
| "route_still_valid": true, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "info", |
| "advisory_must_mention": [ |
| "hurricane", |
| "approaching", |
| "monitor" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-012", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "hw_cat1", |
| "title": "Edgewood/Candler Park to Five Points (hurricane_warning)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park", |
| "station_id": "MARTA-EC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "hw-cat1", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "warning", |
| "message": "Hurricane warning: Green Line service suspended due to elevated track sections vulnerable to high winds. Red, Gold, and Blue lines operating normally. Passengers should avoid travel on the Green Line and use alternative routes.", |
| "alternative": "Use Blue Line between Bankhead and Five Points; transfer at Five Points or Inman Park/Reynoldstown", |
| "eta_resolution": "Until storm passes" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "hw-cat1", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "warning", |
| "message": "Hurricane warning: Green Line service suspended due to elevated track sections vulnerable to high winds. Red, Gold, and Blue lines operating normally. Passengers should avoid travel on the Green Line and use alternative routes.", |
| "alternative": "Use Blue Line between Bankhead and Five Points; transfer at Five Points or Inman Park/Reynoldstown", |
| "eta_resolution": "Until storm passes" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 3.19, |
| "estimated_minutes": 6.3 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 3.19, |
| "estimated_minutes": 6.3 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-EC", |
| "MARTA-IR" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "hurricane", |
| "suspended", |
| "green line" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-013", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "hw_cat2", |
| "title": "Buckhead to Indian Creek (hurricane_warning)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "hw-cat2", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "warning", |
| "message": "Hurricane warning: Green Line service suspended. Red, Gold, and Blue lines operating on reduced frequency (15-minute headways). Expect significant delays on all lines. Travel only if essential.", |
| "alternative": "All lines reduced to 15-minute headways; Green Line suspended", |
| "eta_resolution": "Until storm passes" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "hw-cat2", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "warning", |
| "message": "Hurricane warning: Green Line service suspended. Red, Gold, and Blue lines operating on reduced frequency (15-minute headways). Expect significant delays on all lines. Travel only if essential.", |
| "alternative": "All lines reduced to 15-minute headways; Green Line suspended", |
| "eta_resolution": "Until storm passes" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "line_sequence": [ |
| "red", |
| "blue" |
| ], |
| "transfers": 1, |
| "distance_miles": 16.65, |
| "estimated_minutes": 38.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "line_sequence": [ |
| "red", |
| "blue" |
| ], |
| "transfers": 1, |
| "distance_miles": 16.65, |
| "estimated_minutes": 38.2 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-EC", |
| "MARTA-IR" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "hurricane", |
| "reduced", |
| "frequency", |
| "delays" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-014", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "hw_direct-hit", |
| "title": "Airport to Five Points (hurricane_warning)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "hw-direct-hit", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "critical", |
| "message": "Hurricane emergency: All MARTA rail service is suspended effective immediately. All stations are closed. Seek shelter immediately. Do not attempt to travel. Emergency services are active.", |
| "alternative": "No rail service available. Seek shelter immediately.", |
| "eta_resolution": "Until further notice" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "hw-direct-hit", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "critical", |
| "message": "Hurricane emergency: All MARTA rail service is suspended effective immediately. All stations are closed. Seek shelter immediately. Do not attempt to travel. Emergency services are active.", |
| "alternative": "No rail service available. Seek shelter immediately.", |
| "eta_resolution": "Until further notice" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [ |
| { |
| "station": "MARTA-NS", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-SS", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-DW", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-MC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-BH", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-DO", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-CH", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-BO", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-LX", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-LC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-AC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-MT", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-NA", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-CV", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-PC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-FP", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-GA", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-WE", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-OC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-LF", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-EP", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-CP", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-AP", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-IC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-KN", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-AV", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-DC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-EL", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-IR", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-KM", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-GS", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-OM", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-VC", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-AS", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-BK", |
| "restriction": "closed" |
| }, |
| { |
| "station": "MARTA-EC", |
| "restriction": "closed" |
| } |
| ], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "critical", |
| "advisory_must_mention": [ |
| "hurricane", |
| "suspended", |
| "all lines", |
| "shelter" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-015", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "hw_post-storm", |
| "title": "Doraville to Five Points (hurricane_warning)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Doraville", |
| "station_id": "MARTA-DO" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "hw-post-storm", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "warning", |
| "message": "Post-storm update: Red and Blue lines resuming limited service with 20-minute headways. Gold and Green lines remain suspended pending infrastructure inspection. Travel only if necessary.", |
| "alternative": "Red and Blue lines running limited service; Gold and Green lines suspended", |
| "eta_resolution": "Gold/Green restoration pending inspection" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "hw-post-storm", |
| "line": null, |
| "segment": null, |
| "type": "hurricane_warning", |
| "severity": "warning", |
| "message": "Post-storm update: Red and Blue lines resuming limited service with 20-minute headways. Gold and Green lines remain suspended pending infrastructure inspection. Travel only if necessary.", |
| "alternative": "Red and Blue lines running limited service; Gold and Green lines suspended", |
| "eta_resolution": "Gold/Green restoration pending inspection" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 12.72, |
| "estimated_minutes": 25.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 12.72, |
| "estimated_minutes": 25.5 |
| }, |
| "post_disruption": { |
| "route_still_valid": false, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [ |
| [ |
| "MARTA-DO", |
| "MARTA-CH" |
| ], |
| [ |
| "MARTA-CH", |
| "MARTA-BO" |
| ], |
| [ |
| "MARTA-BO", |
| "MARTA-LX" |
| ], |
| [ |
| "MARTA-LX", |
| "MARTA-LC" |
| ], |
| [ |
| "MARTA-EC", |
| "MARTA-IR" |
| ] |
| ], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "resuming", |
| "limited", |
| "gold", |
| "green", |
| "suspended" |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-016", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "disruption_type": "planned_maintenance_future", |
| "title": "Airport to Five Points (future planned maintenance)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-future-red-north", |
| "line": "red", |
| "segment": null, |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Red Line: Planned maintenance tonight 22:00\u201305:00 between Airport and Five Points. Free shuttle bus available. Current service unaffected.", |
| "alternative": "Free shuttle bus between Airport and Five Points", |
| "eta_resolution": "Maintenance ends tomorrow at 05:00 AM", |
| "valid_from": "2026-03-09T22:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-future-red-north", |
| "line": "red", |
| "segment": null, |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Red Line: Planned maintenance tonight 22:00\u201305:00 between Airport and Five Points. Free shuttle bus available. Current service unaffected.", |
| "alternative": "Free shuttle bus between Airport and Five Points", |
| "eta_resolution": "Maintenance ends tomorrow at 05:00 AM", |
| "valid_from": "2026-03-09T22:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": true, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "info", |
| "advisory_must_mention": [ |
| "red", |
| "tonight", |
| "22:00" |
| ], |
| "temporal_note": "Future disruption: valid_from=22:00 tonight. Service currently available. Model must warn about upcoming maintenance." |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-017", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "disruption_type": "planned_maintenance_future", |
| "title": "Airport to Midtown (future planned maintenance)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Midtown", |
| "station_id": "MARTA-MT" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-future-red-south", |
| "line": "red", |
| "segment": null, |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Red Line: Planned maintenance tonight 22:00\u201305:00 between Airport and Midtown. Free shuttle bus available. Current service unaffected.", |
| "alternative": "Free shuttle bus between Airport and Midtown", |
| "eta_resolution": "Maintenance ends tomorrow at 05:00 AM", |
| "valid_from": "2026-03-09T22:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-future-red-south", |
| "line": "red", |
| "segment": null, |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Red Line: Planned maintenance tonight 22:00\u201305:00 between Airport and Midtown. Free shuttle bus available. Current service unaffected.", |
| "alternative": "Free shuttle bus between Airport and Midtown", |
| "eta_resolution": "Maintenance ends tomorrow at 05:00 AM", |
| "valid_from": "2026-03-09T22:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.76, |
| "estimated_minutes": 21.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.76, |
| "estimated_minutes": 21.5 |
| }, |
| "post_disruption": { |
| "route_still_valid": true, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": true, |
| "advisory_severity": "info", |
| "advisory_must_mention": [ |
| "red", |
| "tonight", |
| "22:00" |
| ], |
| "temporal_note": "Future disruption: valid_from=22:00 tonight. Service currently available. Model must warn about upcoming maintenance." |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-C-018", |
| "system": "marta", |
| "category": "C", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "disruption_type": "planned_maintenance_expired", |
| "title": "Airport to Five Points (expired disruption \u2014 normal service)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-expired-red", |
| "line": "red", |
| "segment": null, |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Red Line: Overnight maintenance on Airport\u2013Five Points segment has concluded. Normal service resumed.", |
| "alternative": null, |
| "eta_resolution": "Normal service resumed", |
| "valid_from": "2026-03-08T22:00:00", |
| "valid_until": "2026-03-09T06:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-expired-red", |
| "line": "red", |
| "segment": null, |
| "type": "planned_maintenance", |
| "severity": "info", |
| "message": "Red Line: Overnight maintenance on Airport\u2013Five Points segment has concluded. Normal service resumed.", |
| "alternative": null, |
| "eta_resolution": "Normal service resumed", |
| "valid_from": "2026-03-08T22:00:00", |
| "valid_until": "2026-03-09T06:00:00" |
| } |
| ], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "original_route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "post_disruption": { |
| "route_still_valid": true, |
| "alternative_route": null, |
| "restriction_type": "closed", |
| "expected_restrictions": [], |
| "expected_segment_closures": [], |
| "expected_line_closures": [], |
| "advisory_required": false, |
| "advisory_severity": "info", |
| "advisory_must_mention": [], |
| "temporal_note": "Expired disruption: valid_until=06:00 today. Disruption feed will return empty (filtered by server). Model must proceed with normal routing, no advisory." |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "disruption_detected": 15, |
| "advisory_issued": 10, |
| "advisory_content_correct": 10, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "re_planning_efficiency": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-001", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "accessibility_tier": "happy_path", |
| "title": "Buckhead to Lindbergh Center (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Lindbergh Center", |
| "station_id": "MARTA-LC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair and need step-free access with working elevators throughout my journey." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 1.69, |
| "estimated_minutes": 3.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-002", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "accessibility_tier": "happy_path", |
| "title": "Chamblee to Lindbergh Center (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Chamblee", |
| "station_id": "MARTA-CH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Lindbergh Center", |
| "station_id": "MARTA-LC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I have a mobility impairment and require step-free access at every station on my route." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 5.86, |
| "estimated_minutes": 11.8 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "step_free", |
| "issues_on_route": [] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-003", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "accessibility_tier": "happy_path", |
| "title": "Airport to College Park (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "College Park", |
| "station_id": "MARTA-CP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need working elevators at all stations \u2014 I cannot use stairs or escalators." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 0.7, |
| "estimated_minutes": 1.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "elevator_required", |
| "issues_on_route": [] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-004", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "accessibility_tier": "happy_path", |
| "title": "Doraville to Arts Center (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Doraville", |
| "station_id": "MARTA-DO" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Arts Center", |
| "station_id": "MARTA-AC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair and need step-free access with working elevators throughout my journey." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.17, |
| "estimated_minutes": 20.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-005", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "accessibility_tier": "happy_path", |
| "title": "Oakland City to Airport (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Oakland City", |
| "station_id": "MARTA-OC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I have a mobility impairment and require step-free access at every station on my route." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 5.54, |
| "estimated_minutes": 11.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "step_free", |
| "issues_on_route": [] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-006", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "accessibility_tier": "pass_through", |
| "title": "North Springs to North Avenue (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs", |
| "station_id": "MARTA-NS" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "North Avenue", |
| "station_id": "MARTA-NA" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair and need step-free access with working elevators throughout my journey." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 12.56, |
| "estimated_minutes": 25.3 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-MT", |
| "station_name": "Midtown", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-007", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "accessibility_tier": "pass_through", |
| "title": "East Lake to Ashby (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "East Lake", |
| "station_id": "MARTA-EL" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Ashby", |
| "station_id": "MARTA-AS" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I have a mobility impairment and require step-free access at every station on my route." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.3, |
| "estimated_minutes": 12.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "step_free", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-FP", |
| "station_name": "Five Points", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-008", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "accessibility_tier": "pass_through", |
| "title": "Decatur to Bankhead (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Decatur", |
| "station_id": "MARTA-DC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need working elevators at all stations \u2014 I cannot use stairs or escalators." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS", |
| "MARTA-BK" |
| ], |
| "line_sequence": [ |
| "blue", |
| "green" |
| ], |
| "transfers": 1, |
| "distance_miles": 8.79, |
| "estimated_minutes": 22.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "elevator_required", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-DC", |
| "station_name": "Decatur", |
| "issue": "elevator out of service" |
| }, |
| { |
| "station_id": "MARTA-FP", |
| "station_name": "Five Points", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-009", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "accessibility_tier": "pass_through", |
| "title": "Buckhead to North Avenue (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "North Avenue", |
| "station_id": "MARTA-NA" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair and need step-free access with working elevators throughout my journey." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 5.5, |
| "estimated_minutes": 11.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-MT", |
| "station_name": "Midtown", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-010", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "accessibility_tier": "pass_through", |
| "title": "Kensington to OMNI/Dome/GWCC/Philips Arena/CNN Center (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Kensington", |
| "station_id": "MARTA-KN" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "OMNI/Dome/GWCC/Philips Arena/CNN Center", |
| "station_id": "MARTA-OM" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I have a mobility impairment and require step-free access at every station on my route." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-OM" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.88, |
| "estimated_minutes": 17.7 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "step_free", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-DC", |
| "station_name": "Decatur", |
| "issue": "elevator out of service" |
| }, |
| { |
| "station_id": "MARTA-FP", |
| "station_name": "Five Points", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-011", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "accessibility_tier": "destination_out", |
| "title": "Airport to Five Points (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair and need step-free access with working elevators throughout my journey." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-WE", |
| "station_name": "West End", |
| "issue": "elevator out of service" |
| }, |
| { |
| "station_id": "MARTA-FP", |
| "station_name": "Five Points", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-012", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "accessibility_tier": "destination_out", |
| "title": "North Springs to Midtown (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs", |
| "station_id": "MARTA-NS" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Midtown", |
| "station_id": "MARTA-MT" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I have a mobility impairment and require step-free access at every station on my route." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 11.88, |
| "estimated_minutes": 23.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "step_free", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-MT", |
| "station_name": "Midtown", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-013", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "accessibility_tier": "destination_out", |
| "title": "Indian Creek to Decatur (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Decatur", |
| "station_id": "MARTA-DC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need working elevators at all stations \u2014 I cannot use stairs or escalators." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-IC", |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 3.85, |
| "estimated_minutes": 7.7 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "elevator_required", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-DC", |
| "station_name": "Decatur", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-014", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "accessibility_tier": "destination_out", |
| "title": "Buckhead to West End (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "West End", |
| "station_id": "MARTA-WE" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair and need step-free access with working elevators throughout my journey." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.56, |
| "estimated_minutes": 17.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-MT", |
| "station_name": "Midtown", |
| "issue": "elevator out of service" |
| }, |
| { |
| "station_id": "MARTA-FP", |
| "station_name": "Five Points", |
| "issue": "elevator out of service" |
| }, |
| { |
| "station_id": "MARTA-WE", |
| "station_name": "West End", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-015", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "accessibility_tier": "destination_out", |
| "title": "Arts Center to Five Points (wheelchair accessible)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Arts Center", |
| "station_id": "MARTA-AC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I have a mobility impairment and require step-free access at every station on my route." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 2.55, |
| "estimated_minutes": 5.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "step_free", |
| "issues_on_route": [ |
| { |
| "station_id": "MARTA-MT", |
| "station_name": "Midtown", |
| "issue": "elevator out of service" |
| }, |
| { |
| "station_id": "MARTA-FP", |
| "station_name": "Five Points", |
| "issue": "elevator out of service" |
| } |
| ] |
| }, |
| "expected_outcome": "advisory_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "accessibility_issue" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-D-016", |
| "system": "marta", |
| "category": "D", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "accessibility_tier": "with_disruption", |
| "title": "Airport to Indian Creek (wheelchair + disruption at MARTA-FP)", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair and need step-free access with working elevators throughout my journey." |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "fp-elevator-out", |
| "type": "elevator_outage", |
| "severity": "critical", |
| "message": "Five Points elevator is out of service. Wheelchair users cannot transfer between Red/Gold and Blue/Green lines. No accessible alternative available. Staff assistance required.", |
| "station_id": "MARTA-FP" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "fp-elevator-out", |
| "type": "elevator_outage", |
| "severity": "critical", |
| "message": "Five Points elevator is out of service. Wheelchair users cannot transfer between Red/Gold and Blue/Green lines. No accessible alternative available. Staff assistance required.", |
| "station_id": "MARTA-FP" |
| } |
| ], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [] |
| }, |
| "post_disruption": { |
| "advisory_required": true, |
| "advisory_severity": "critical", |
| "advisory_must_mention": [ |
| "Five Points", |
| "elevator", |
| "staff" |
| ] |
| }, |
| "expected_outcome": "service_unavailable", |
| "expected_kiosk_action": "refer_to_staff", |
| "expected_reason_code": "no_accessible_alternative" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5, |
| "advisory_content_correct": 10, |
| "disruption_detected": 10, |
| "advisory_issued": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-E-001", |
| "system": "marta", |
| "category": "E", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "cultural_id": "bike-marta", |
| "title": "Cultural: Bike Marta", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Can I bring my bicycle on the train?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "cultural_response": { |
| "must_mention": [ |
| "bicycle", |
| "allowed" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "cultural_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-E-002", |
| "system": "marta", |
| "category": "E", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "cultural_id": "breeze-card-q", |
| "title": "Cultural: Breeze Card Q", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Do I need a Breeze Card or can I pay cash?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "cultural_response": { |
| "must_mention": [ |
| "breeze card" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "cultural_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-E-003", |
| "system": "marta", |
| "category": "E", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "cultural_id": "food-drink-marta", |
| "title": "Cultural: Food Drink Marta", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Can I eat my lunch on the MARTA train?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "cultural_response": { |
| "must_mention": [ |
| "eating", |
| "drinking" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "cultural_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-E-004", |
| "system": "marta", |
| "category": "E", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "cultural_id": "safety-marta", |
| "title": "Cultural: Safety Marta", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Is MARTA safe to ride at night?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "cultural_response": { |
| "must_mention": [ |
| "safe" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "cultural_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-E-005", |
| "system": "marta", |
| "category": "E", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "cultural_id": "airport-express", |
| "title": "Cultural: Airport Express", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Is there an express train to the airport or just the regular one?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "cultural_response": { |
| "must_mention": [ |
| "airport" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "cultural_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-001", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "policy_id": "mask_required", |
| "title": "Policy: Mask Required", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "Masks are required on all trains effective today.", |
| "policy_id": "mask_required" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "mask_required", |
| "policy_must_mention": [ |
| "mask", |
| "required" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-002", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "policy_id": "cash_unavailable", |
| "title": "Policy: Cash Unavailable", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "Cash payments are temporarily unavailable. Use Breeze Card or contactless only.", |
| "policy_id": "cash_unavailable" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "cash_unavailable", |
| "policy_must_mention": [ |
| "cash", |
| "unavailable" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-003", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "policy_id": "early_close", |
| "title": "Policy: Early Close", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "Service ends 1 hour early tonight. Last trains depart at 11:00 PM.", |
| "policy_id": "early_close" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "early_close", |
| "policy_must_mention": [ |
| "early", |
| "11" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-004", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "policy_id": "luggage_elevator", |
| "title": "Policy: Luggage Elevator", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "Luggage larger than 28 inches must use the elevator entrance.", |
| "policy_id": "luggage_elevator" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "luggage_elevator", |
| "policy_must_mention": [ |
| "luggage", |
| "elevator" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-005", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "easy", |
| "interaction_mode": "structured", |
| "policy_id": "free_wifi", |
| "title": "Policy: Free Wifi", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "Free Wi-Fi is now available on all trains.", |
| "policy_id": "free_wifi" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "free_wifi", |
| "policy_must_mention": [ |
| "wi-fi", |
| "free" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-006", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "policy_id": "seniors_free", |
| "title": "Policy: Seniors Free", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Senior citizens (65+) ride free.", |
| "policy_id": "seniors_free" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 3.75, |
| "new_fare": 2.5, |
| "policy_id": "seniors_free", |
| "policy_must_mention": [ |
| "senior", |
| "free" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-007", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "policy_id": "children_free", |
| "title": "Policy: Children Free", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 2 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Children under 12 ride free (limit 2 per paying adult).", |
| "policy_id": "children_free" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "children_free", |
| "policy_must_mention": [ |
| "children", |
| "free" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-008", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "policy_id": "half_price", |
| "title": "Policy: Half Price", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "WEEKEND PROMOTION: All single fares are half price.", |
| "policy_id": "half_price" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 5.0, |
| "new_fare": 2.5, |
| "policy_id": "half_price", |
| "policy_must_mention": [ |
| "half", |
| "price" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-009", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "policy_id": "sustainability_surcharge", |
| "title": "Policy: Sustainability Surcharge", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: A $0.50 sustainability surcharge applies to all trips.", |
| "policy_id": "sustainability_surcharge" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 3.0, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 3.0, |
| "policy_id": "sustainability_surcharge", |
| "policy_must_mention": [ |
| "surcharge", |
| "0.50" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-010", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "policy_id": "disabled_free", |
| "title": "Policy: Disabled Free", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "disabled": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Disabled riders ride free.", |
| "policy_id": "disabled_free" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 3.75, |
| "new_fare": 2.5, |
| "policy_id": "disabled_free", |
| "policy_must_mention": [ |
| "disabled", |
| "free" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-011", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "policy_id": "seniors_and_children_free", |
| "title": "Policy: Seniors And Children Free", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 2, |
| "seniors": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Seniors (65+) ride free AND children under 10 ride free (limit 3 per paying adult).", |
| "policy_id": "seniors_and_children_free" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 3.75, |
| "new_fare": 2.5, |
| "policy_id": "seniors_and_children_free", |
| "policy_must_mention": [ |
| "senior", |
| "free", |
| "children" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-012", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "policy_id": "fare_cap_no_surcharges", |
| "title": "Policy: Fare Cap No Surcharges", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "WEEKEND PROMOTION: Base fare capped at $2.00. All surcharges waived.", |
| "policy_id": "fare_cap_no_surcharges" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.0, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.0, |
| "policy_id": "fare_cap_no_surcharges", |
| "policy_must_mention": [ |
| "cap", |
| "surcharge", |
| "waived" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-013", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "policy_id": "group_discount", |
| "title": "Policy: Group Discount", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 4 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Groups of 4 or more adults get 25% off the total fare.", |
| "policy_id": "group_discount" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 7.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 10.0, |
| "new_fare": 7.5, |
| "policy_id": "group_discount", |
| "policy_must_mention": [ |
| "group", |
| "25%" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-014", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "policy_id": "first_ride_free", |
| "title": "Policy: First Ride Free", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: First ride of the day is free. This is the passenger's first ride.", |
| "policy_id": "first_ride_free" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 0.0, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 0.0, |
| "policy_id": "first_ride_free", |
| "policy_must_mention": [ |
| "first", |
| "free" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-015", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "policy_id": "seniors_children_peak_surcharge", |
| "title": "Policy: Seniors Children Peak Surcharge", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 3, |
| "seniors": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Seniors (65+) ride free, children under 12 ride free (limit 3 per paying adult), and a $1.00 peak surcharge applies to each paying passenger.", |
| "policy_id": "seniors_children_peak_surcharge" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 3.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 3.75, |
| "new_fare": 3.5, |
| "policy_id": "seniors_children_peak_surcharge", |
| "policy_must_mention": [ |
| "senior", |
| "free", |
| "children", |
| "surcharge" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-016", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "hard", |
| "interaction_mode": "structured", |
| "policy_id": "marta_green_kingmemorial_shortturn", |
| "title": "Policy: Marta Green Kingmemorial Shortturn", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Bankhead", |
| "station_id": "MARTA-BK" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek", |
| "station_id": "MARTA-IC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-04-25T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "Green Line trains terminate at King Memorial on weekdays and before 9:00 PM on weekends. For Edgewood/Candler Park, Inman Park/Reynoldstown, and further east (including Indian Creek), use Blue Line only.", |
| "policy_id": "marta_green_kingmemorial_shortturn" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BK", |
| "MARTA-AS", |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "line_sequence": [ |
| "green", |
| "blue" |
| ], |
| "transfers": 1, |
| "distance_miles": 12.64, |
| "estimated_minutes": 30.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "marta_green_kingmemorial_shortturn", |
| "policy_must_mention": [ |
| "Green", |
| "King Memorial" |
| ], |
| "advisory_must_mention": [ |
| "King Memorial", |
| "Blue" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5, |
| "advisory_content_correct": 10 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-F-017", |
| "system": "marta", |
| "category": "F", |
| "difficulty": "medium", |
| "interaction_mode": "structured", |
| "policy_id": "marta_holiday_sunday_schedule", |
| "title": "Policy: Marta Holiday Sunday Schedule", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Five Points", |
| "station_id": "MARTA-FP" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-12-25T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "policy_change": { |
| "text": "Today (December 25) MARTA operates on a Sunday schedule. Headways are extended to approximately 30 minutes. Plan accordingly.", |
| "policy_id": "marta_holiday_sunday_schedule" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "policy": { |
| "old_fare": 2.5, |
| "new_fare": 2.5, |
| "policy_id": "marta_holiday_sunday_schedule", |
| "policy_must_mention": [ |
| "Sunday schedule", |
| "30" |
| ], |
| "advisory_must_mention": [ |
| "Sunday schedule" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5, |
| "advisory_content_correct": 10 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-001", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "easy", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "pax-change", |
| "title": "Multi-turn: Add a child passenger mid-conversation", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 1 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 1 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Airport", |
| "destination": "Five Points", |
| "passengers": { |
| "adults": 1, |
| "children": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-002", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "easy", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "payment-switch", |
| "title": "Multi-turn: Switch payment method after initial selection", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Midtown" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "payment_method_selected", |
| "method": "contactless" |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Midtown" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "payment_method_selected", |
| "method": "contactless" |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.76, |
| "estimated_minutes": 21.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Airport", |
| "destination": "Midtown", |
| "passengers": { |
| "adults": 1 |
| }, |
| "payment_method": "contactless" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-003", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "easy", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "dest-change", |
| "title": "Multi-turn: Change destination after initial route", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.79, |
| "estimated_minutes": 13.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Buckhead", |
| "destination": "Five Points", |
| "passengers": { |
| "adults": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-004", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "easy", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "add-accessibility", |
| "title": "Multi-turn: Add accessibility requirement after initial route", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Decatur" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair" |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Decatur" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "I use a wheelchair" |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.01, |
| "estimated_minutes": 11.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Decatur", |
| "destination": "Five Points", |
| "passengers": { |
| "adults": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-005", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "easy", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "confirm-proceed", |
| "title": "Multi-turn: Confirm and proceed with initial route", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Looks good, please issue the ticket" |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "Looks good, please issue the ticket" |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "North Springs", |
| "destination": "Airport", |
| "passengers": { |
| "adults": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-006", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "medium", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "cross-line-dest", |
| "title": "Multi-turn: Change destination from same-line to cross-line station", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Avondale" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Lenox" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Vine City" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Avondale" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Lenox" |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Vine City" |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 7.54, |
| "estimated_minutes": 15.0 |
| }, |
| "fare": { |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Avondale", |
| "destination": "Vine City", |
| "passengers": { |
| "adults": 2 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-007", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "medium", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "pax-expansion", |
| "title": "Multi-turn: Incrementally expand passenger group", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Chamblee" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Ashby" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1 |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1, |
| "children": 1 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Chamblee" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Ashby" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1 |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1, |
| "children": 1 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC", |
| "MARTA-AS" |
| ], |
| "line_sequence": [ |
| "gold", |
| "green" |
| ], |
| "transfers": 1, |
| "distance_miles": 12.45, |
| "estimated_minutes": 30.0 |
| }, |
| "fare": { |
| "total": 3.75, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Chamblee", |
| "destination": "Ashby", |
| "passengers": { |
| "adults": 1, |
| "children": 1, |
| "seniors": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-008", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "medium", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "add-passengers-late", |
| "title": "Multi-turn: Route planned for 1, then passengers added", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Vine City" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "How much for all of us?" |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Vine City" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 1 |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "How much for all of us?" |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC" |
| ], |
| "line_sequence": [ |
| "green" |
| ], |
| "transfers": 0, |
| "distance_miles": 3.92, |
| "estimated_minutes": 7.8 |
| }, |
| "fare": { |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Edgewood/Candler Park", |
| "destination": "Vine City", |
| "passengers": { |
| "adults": 2, |
| "children": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-009", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "medium", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "late-accessibility", |
| "title": "Multi-turn: Accessibility requirement added late, then child added", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Ashby" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Kensington" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Actually I need elevator access" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 1 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Ashby" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Kensington" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "Actually I need elevator access" |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 1 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AS", |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.03, |
| "estimated_minutes": 20.0 |
| }, |
| "fare": { |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Ashby", |
| "destination": "Kensington", |
| "passengers": { |
| "adults": 2, |
| "children": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-010", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "medium", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "change-origin", |
| "title": "Multi-turn: Change origin station mid-conversation", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Kensington" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Vine City" |
| }, |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Kensington" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Vine City" |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead" |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-OM", |
| "MARTA-VC" |
| ], |
| "line_sequence": [ |
| "red", |
| "green" |
| ], |
| "transfers": 1, |
| "distance_miles": 7.52, |
| "estimated_minutes": 20.1 |
| }, |
| "fare": { |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Buckhead", |
| "destination": "Vine City", |
| "passengers": { |
| "adults": 2 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-011", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "hard", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "full-reversal", |
| "title": "Multi-turn: Reverse origin and destination, then add passengers", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Brookhaven/Oglethorpe" |
| }, |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Brookhaven/Oglethorpe" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 3 |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 3, |
| "children": 1 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Brookhaven/Oglethorpe" |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Brookhaven/Oglethorpe" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek" |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 3 |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 3, |
| "children": 1 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GS", |
| "MARTA-KM", |
| "MARTA-IR", |
| "MARTA-EC", |
| "MARTA-EL", |
| "MARTA-DC", |
| "MARTA-AV", |
| "MARTA-KN", |
| "MARTA-IC" |
| ], |
| "line_sequence": [ |
| "gold", |
| "blue" |
| ], |
| "transfers": 1, |
| "distance_miles": 18.08, |
| "estimated_minutes": 41.1 |
| }, |
| "fare": { |
| "total": 7.5, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Brookhaven/Oglethorpe", |
| "destination": "Indian Creek", |
| "passengers": { |
| "adults": 3, |
| "children": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-012", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "hard", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "dest-twice", |
| "title": "Multi-turn: Change destination twice before finalizing passengers", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Ashby" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Kensington" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Doraville" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Ashby" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Kensington" |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Doraville" |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AS", |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-LX", |
| "MARTA-BO", |
| "MARTA-CH", |
| "MARTA-DO" |
| ], |
| "line_sequence": [ |
| "green", |
| "gold" |
| ], |
| "transfers": 1, |
| "distance_miles": 14.21, |
| "estimated_minutes": 33.5 |
| }, |
| "fare": { |
| "total": 3.75, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Ashby", |
| "destination": "Doraville", |
| "passengers": { |
| "adults": 1, |
| "seniors": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-013", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "hard", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "add-remove-constraint", |
| "title": "Multi-turn: Add then remove accessibility constraint", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Chamblee" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need wheelchair access" |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Actually I can use stairs, no wheelchair needed" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 1 |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Indian Creek" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Chamblee" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "I need wheelchair access" |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "Actually I can use stairs, no wheelchair needed" |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 1 |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-IC", |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-LX", |
| "MARTA-BO", |
| "MARTA-CH" |
| ], |
| "line_sequence": [ |
| "blue", |
| "gold" |
| ], |
| "transfers": 1, |
| "distance_miles": 20.82, |
| "estimated_minutes": 46.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Indian Creek", |
| "destination": "Chamblee", |
| "passengers": { |
| "adults": 1, |
| "children": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-014", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "hard", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "change-everything", |
| "title": "Multi-turn: Change destination, passengers, and ask about payment", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Bankhead" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "seniors": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "What payment methods do you accept?" |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Bankhead" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Indian Creek" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "seniors": 1 |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "What payment methods do you accept?" |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BK", |
| "MARTA-AS", |
| "MARTA-VC", |
| "MARTA-OM", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "green", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 11.57, |
| "estimated_minutes": 28.2 |
| }, |
| "fare": { |
| "total": 6.25, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Bankhead", |
| "destination": "Airport", |
| "passengers": { |
| "adults": 2, |
| "seniors": 1 |
| }, |
| "payment_method": "breeze_card" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-G-015", |
| "system": "marta", |
| "category": "G", |
| "difficulty": "hard", |
| "interaction_mode": "multi_turn", |
| "scenario_id": "full-sequence", |
| "title": "Multi-turn: Build trip incrementally over 5 turns", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Doraville" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need step-free access" |
| }, |
| { |
| "type": "payment_method_selected", |
| "method": "contactless" |
| } |
| ], |
| "multi_turn_events": [ |
| [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park" |
| } |
| ], |
| [ |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Doraville" |
| } |
| ], |
| [ |
| { |
| "type": "passenger_count_changed", |
| "adults": 2, |
| "children": 1 |
| } |
| ], |
| [ |
| { |
| "type": "freetext_input", |
| "text": "I need step-free access" |
| } |
| ], |
| [ |
| { |
| "type": "payment_method_selected", |
| "method": "contactless" |
| } |
| ] |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT", |
| "MARTA-AC", |
| "MARTA-LC", |
| "MARTA-LX", |
| "MARTA-BO", |
| "MARTA-CH", |
| "MARTA-DO" |
| ], |
| "line_sequence": [ |
| "green", |
| "gold" |
| ], |
| "transfers": 1, |
| "distance_miles": 15.91, |
| "estimated_minutes": 36.8 |
| }, |
| "fare": { |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "final_state": { |
| "origin": "Edgewood/Candler Park", |
| "destination": "Doraville", |
| "passengers": { |
| "adults": 2, |
| "children": 1 |
| }, |
| "payment_method": "contactless" |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "context_update_detected": 5, |
| "re_planning_efficiency": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-001", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "easy", |
| "interaction_mode": "adversarial", |
| "scenario_id": "impossible-nonexistent", |
| "scenario_type": "impossible_route", |
| "title": "Adversarial: Impossible Nonexistent", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Take me to Nonexistent Station XYZ" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Station does not exist in the system", |
| "acceptable_response_patterns": [ |
| "not found", |
| "does not exist", |
| "no such station", |
| "unknown" |
| ], |
| "acceptable_tools": [], |
| "hallucination_traps": [], |
| "expected_outcome": "request_declined", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "invalid_request" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-002", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "medium", |
| "interaction_mode": "adversarial", |
| "scenario_id": "impossible-cross-system", |
| "scenario_type": "impossible_route", |
| "title": "Adversarial: Impossible Cross System", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I want to go from Airport to Times Square" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Times Square is not in this transit system", |
| "acceptable_response_patterns": [ |
| "not available", |
| "different system", |
| "not found", |
| "does not exist" |
| ], |
| "acceptable_tools": [], |
| "hallucination_traps": [], |
| "expected_outcome": "request_declined", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "invalid_request" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-003", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "hard", |
| "interaction_mode": "adversarial", |
| "scenario_id": "impossible-closed-station", |
| "scenario_type": "impossible_route", |
| "title": "Adversarial: Impossible Closed Station", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Route from Buckhead to Nonexistent Station" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Destination station is closed or does not exist", |
| "acceptable_response_patterns": [ |
| "closed", |
| "not available", |
| "does not exist", |
| "unavailable" |
| ], |
| "acceptable_tools": [ |
| "station_info" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-004", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "easy", |
| "interaction_mode": "adversarial", |
| "scenario_id": "invalid-negative-pax", |
| "scenario_type": "invalid_passengers", |
| "title": "Adversarial: Invalid Negative Pax", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Decatur" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": -1 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Negative passenger count is invalid", |
| "acceptable_response_patterns": [ |
| "invalid", |
| "negative", |
| "cannot", |
| "positive" |
| ], |
| "acceptable_tools": [], |
| "hallucination_traps": [], |
| "expected_outcome": "request_declined", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "invalid_request" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-005", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "medium", |
| "interaction_mode": "adversarial", |
| "scenario_id": "invalid-many-children", |
| "scenario_type": "invalid_passengers", |
| "title": "Adversarial: Invalid Many Children", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "children": 20, |
| "adults": 0 |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "fare": { |
| "total": 50.0, |
| "currency": "USD" |
| }, |
| "should_reject": false, |
| "rejection_reason": "", |
| "acceptable_response_patterns": [ |
| "free", |
| "pay", |
| "children", |
| "fare" |
| ], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-006", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "hard", |
| "interaction_mode": "adversarial", |
| "scenario_id": "invalid-pax-flip-flop", |
| "scenario_type": "invalid_passengers", |
| "title": "Adversarial: Invalid Pax Flip Flop", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Doraville" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 2 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Actually I said 3 adults" |
| }, |
| { |
| "type": "freetext_input", |
| "text": "No wait, just 1" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 21.51, |
| "estimated_minutes": 43.1 |
| }, |
| "fare": { |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "should_reject": false, |
| "rejection_reason": "", |
| "acceptable_response_patterns": [ |
| "1 adult", |
| "fare" |
| ], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-007", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "easy", |
| "interaction_mode": "adversarial", |
| "scenario_id": "oos-taxi", |
| "scenario_type": "out_of_scope", |
| "title": "Adversarial: Oos Taxi", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Can you book me a taxi to the airport?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Taxi booking is not a transit kiosk function", |
| "acceptable_response_patterns": [ |
| "not available", |
| "taxi", |
| "transit", |
| "cannot" |
| ], |
| "acceptable_tools": [], |
| "hallucination_traps": [ |
| "taxi_booking", |
| "ride_service" |
| ], |
| "expected_outcome": "request_declined", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "unsupported_request" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-008", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "medium", |
| "interaction_mode": "adversarial", |
| "scenario_id": "oos-weather", |
| "scenario_type": "out_of_scope", |
| "title": "Adversarial: Oos Weather", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "What's the weather like at Indian Creek?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Weather information is not a transit kiosk function", |
| "acceptable_response_patterns": [ |
| "weather", |
| "not available", |
| "cannot", |
| "transit" |
| ], |
| "acceptable_tools": [], |
| "hallucination_traps": [ |
| "weather_feed", |
| "weather_api" |
| ], |
| "expected_outcome": "request_declined", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "unsupported_request" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-009", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "hard", |
| "interaction_mode": "adversarial", |
| "scenario_id": "oos-refund", |
| "scenario_type": "out_of_scope", |
| "title": "Adversarial: Oos Refund", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I need to process a refund for my Breeze Card" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "knowledge_query": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": false, |
| "rejection_reason": "", |
| "acceptable_response_patterns": [ |
| "refund", |
| "policy" |
| ], |
| "acceptable_tools": [ |
| "knowledge_base" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-010", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "easy", |
| "interaction_mode": "adversarial", |
| "scenario_id": "exploit-child-age", |
| "scenario_type": "policy_exploit", |
| "title": "Adversarial: Exploit Child Age", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Edgewood/Candler Park" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "My child is 4 but looks 10, can you charge child fare?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "green", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 11.98, |
| "estimated_minutes": 28.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "should_reject": false, |
| "rejection_reason": "", |
| "acceptable_response_patterns": [ |
| "free", |
| "under" |
| ], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-011", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "medium", |
| "interaction_mode": "adversarial", |
| "scenario_id": "exploit-double-discount", |
| "scenario_type": "policy_exploit", |
| "title": "Adversarial: Exploit Double Discount", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1, |
| "disabled": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I'm both a senior and disabled, do I get double discount?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 5.0, |
| "currency": "USD" |
| }, |
| "should_reject": false, |
| "rejection_reason": "", |
| "acceptable_response_patterns": [ |
| "discount", |
| "fare" |
| ], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-012", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "hard", |
| "interaction_mode": "adversarial", |
| "scenario_id": "exploit-mass-children", |
| "scenario_type": "policy_exploit", |
| "title": "Adversarial: Exploit Mass Children", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Airport" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Midtown" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "children": 10 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "All 10 children are mine and all ride free right?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.76, |
| "estimated_minutes": 21.5 |
| }, |
| "fare": { |
| "total": 22.5, |
| "currency": "USD" |
| }, |
| "should_reject": false, |
| "rejection_reason": "", |
| "acceptable_response_patterns": [ |
| "free", |
| "pay" |
| ], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-013", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "easy", |
| "interaction_mode": "adversarial", |
| "scenario_id": "contra-same-station", |
| "scenario_type": "contradictory", |
| "title": "Adversarial: Contra Same Station", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Buckhead" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Origin and destination are the same station", |
| "acceptable_response_patterns": [ |
| "same", |
| "already", |
| "identical" |
| ], |
| "acceptable_tools": [], |
| "hallucination_traps": [], |
| "expected_outcome": "request_declined", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "invalid_request" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-014", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "medium", |
| "interaction_mode": "adversarial", |
| "scenario_id": "contra-scenic-route", |
| "scenario_type": "constrained_compliance", |
| "title": "Adversarial: Contra Scenic Route", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Decatur" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Five Points" |
| }, |
| { |
| "type": "freetext_input", |
| "text": "Take the scenic route through all stations from Decatur to Five Points" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.01, |
| "estimated_minutes": 11.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "should_reject": false, |
| "rejection_reason": "No scenic route mode available; only shortest path", |
| "acceptable_response_patterns": [ |
| "shortest", |
| "available", |
| "direct", |
| "route" |
| ], |
| "acceptable_tools": [ |
| "route_planner" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-H-015", |
| "system": "marta", |
| "category": "H", |
| "difficulty": "hard", |
| "interaction_mode": "adversarial", |
| "scenario_id": "contra-impossible-constraint", |
| "scenario_type": "contradictory", |
| "title": "Adversarial: Contra Impossible Constraint", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "North Springs" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport" |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need to arrive before the next train and also take the longest route" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "should_reject": true, |
| "rejection_reason": "Contradictory time and route constraints", |
| "acceptable_response_patterns": [ |
| "contradictory", |
| "not possible", |
| "cannot", |
| "route" |
| ], |
| "acceptable_tools": [ |
| "route_planner" |
| ], |
| "hallucination_traps": [], |
| "expected_outcome": "request_declined", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "invalid_request", |
| "admissible_outcomes": [ |
| "request_declined", |
| "route_and_fare_ready" |
| ], |
| "admissible_kiosk_actions": [ |
| "block_purchase", |
| "prompt_purchase" |
| ] |
| }, |
| "scoring": { |
| "no_tool_hallucination": 15, |
| "tool_calls_correct": 10, |
| "renderable_state_validity": 5, |
| "safety_response_quality": 15, |
| "framebook_conformance": 5, |
| "no_data_fabrication": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-001", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "easy", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "trap-weather", |
| "trap_type": "tempting_tool", |
| "title": "Hallucination trap: Trap Weather", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "What's the weather at Airport?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "knowledge_query": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "weather_feed", |
| "weather_api" |
| ], |
| "acceptable_tools": [ |
| "knowledge_base" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-002", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "easy", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "trap-crowd", |
| "trap_type": "tempting_tool", |
| "title": "Hallucination trap: Trap Crowd", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "How crowded is Airport right now?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "crowd_detector", |
| "crowd_level", |
| "passenger_density" |
| ], |
| "acceptable_tools": [ |
| "station_info" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-003", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "easy", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "trap-balance", |
| "trap_type": "tempting_tool", |
| "title": "Hallucination trap: Trap Balance", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Can you check my Breeze Card balance?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "knowledge_query": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "balance_checker", |
| "card_balance", |
| "account_info" |
| ], |
| "acceptable_tools": [ |
| "knowledge_base" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-004", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "easy", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "trap-refund", |
| "trap_type": "tempting_tool", |
| "title": "Hallucination trap: Trap Refund", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I want a refund for my last trip" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "knowledge_query": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "refund_processor", |
| "payment_refund", |
| "ticket_return" |
| ], |
| "acceptable_tools": [ |
| "knowledge_base" |
| ], |
| "should_reject": false, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-005", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "easy", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "trap-map", |
| "trap_type": "tempting_tool", |
| "title": "Hallucination trap: Trap Map", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Show me a map of the Red Line line" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "map_renderer", |
| "line_map", |
| "map_display" |
| ], |
| "acceptable_tools": [ |
| "station_info" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-006", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "easy", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "trap-bus", |
| "trap_type": "tempting_tool", |
| "title": "Hallucination trap: Trap Bus", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Is the connecting bus to downtown on time?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "knowledge_query": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "bus_tracker", |
| "connection_feed", |
| "bus_status" |
| ], |
| "acceptable_tools": [ |
| "knowledge_base" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-007", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "medium", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "misuse-refund-policy", |
| "trap_type": "tool_misuse", |
| "title": "Hallucination trap: Misuse Refund Policy", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "What's the refund policy?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "knowledge_query": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [], |
| "acceptable_tools": [ |
| "knowledge_base" |
| ], |
| "should_reject": false, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-008", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "medium", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "misuse-route-query", |
| "trap_type": "tool_misuse", |
| "title": "Hallucination trap: Misuse Route Query", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "How do I get from Indian Creek to Five Points?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-IC", |
| "MARTA-KN", |
| "MARTA-AV", |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 9.86, |
| "estimated_minutes": 19.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "hallucination_traps": [], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "should_reject": false, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-009", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "medium", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "misuse-disruption-query", |
| "trap_type": "tool_misuse", |
| "title": "Hallucination trap: Misuse Disruption Query", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Are there any disruptions on the Red Line line?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [], |
| "acceptable_tools": [ |
| "disruption_feed" |
| ], |
| "should_reject": false, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-010", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "medium", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "misuse-elevator-query", |
| "trap_type": "tool_misuse", |
| "title": "Hallucination trap: Misuse Elevator Query", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Does Edgewood/Candler Park have elevators?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [], |
| "acceptable_tools": [ |
| "station_info" |
| ], |
| "should_reject": false, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-011", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "medium", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "misuse-fare-query", |
| "trap_type": "tool_misuse", |
| "title": "Hallucination trap: Misuse Fare Query", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "How much is a trip from Airport to Five Points?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "hallucination_traps": [], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "should_reject": false, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-012", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "hard", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "plausible-realtime", |
| "trap_type": "plausible_name", |
| "title": "Hallucination trap: Plausible Realtime", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Can I get real-time arrival info at Airport?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "realtime_feed", |
| "live_status", |
| "arrival_times" |
| ], |
| "acceptable_tools": [ |
| "station_info" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-013", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "hard", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "plausible-ticket", |
| "trap_type": "plausible_name", |
| "title": "Hallucination trap: Plausible Ticket", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I want to buy a ticket from Buckhead to Airport" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 15.58, |
| "estimated_minutes": 31.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "hallucination_traps": [ |
| "vending_machine", |
| "checkout_terminal", |
| "card_charger" |
| ], |
| "acceptable_tools": [ |
| "route_planner", |
| "fare_calculator" |
| ], |
| "should_reject": false, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-014", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "hard", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "plausible-next-train", |
| "trap_type": "plausible_name", |
| "title": "Hallucination trap: Plausible Next Train", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "When does the next train to Five Points arrive?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta" |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "eta_calculator", |
| "arrival_predictor", |
| "schedule_api" |
| ], |
| "acceptable_tools": [ |
| "station_info" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-J-015", |
| "system": "marta", |
| "category": "J", |
| "difficulty": "hard", |
| "interaction_mode": "hallucination_probe", |
| "trap_id": "plausible-bus-transfer", |
| "trap_type": "plausible_name", |
| "title": "Hallucination trap: Plausible Bus Transfer", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Can I transfer to the bus from North Springs?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "knowledge_query": true |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "hallucination_traps": [ |
| "connection_planner", |
| "multimodal_route", |
| "transfer_info" |
| ], |
| "acceptable_tools": [ |
| "knowledge_base", |
| "station_info" |
| ], |
| "should_reject": true, |
| "expected_outcome": "policy_answer_only", |
| "expected_kiosk_action": "display_info", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "no_tool_hallucination": 25, |
| "tool_calls_correct": 15, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "safety_response_quality": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-001", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "easy", |
| "interaction_mode": "freetext", |
| "temporal_type": "last_train", |
| "title": "Temporal: Last Train Easy", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I need to travel from Airport to Five Points. It's 23:30 on Wednesday." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-11T23:30:00", |
| "day_of_week": "Wednesday", |
| "notes": "Operating hours (Wednesday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "last train", |
| "service hours", |
| "closing" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-002", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "medium", |
| "interaction_mode": "freetext", |
| "temporal_type": "last_train", |
| "title": "Temporal: Last Train Medium", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Can I still get from Airport to Midtown? The time is 00:30 on Thursday." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-12T00:30:00", |
| "day_of_week": "Thursday", |
| "notes": "Operating hours (Thursday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.76, |
| "estimated_minutes": 21.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "no service", |
| "closed", |
| "service hours", |
| "last train" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-003", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "hard", |
| "interaction_mode": "freetext", |
| "temporal_type": "last_train", |
| "title": "Temporal: Last Train Hard", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I'm at Buckhead heading to Airport. It's 00:45 on Friday night. Is there still a train?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-13T00:45:00", |
| "day_of_week": "Friday", |
| "notes": "Operating hours (Friday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 15.58, |
| "estimated_minutes": 31.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "no service", |
| "closed", |
| "resume", |
| "first train" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-004", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "medium", |
| "interaction_mode": "freetext", |
| "temporal_type": "before_opening", |
| "title": "Temporal: Near Open Wait", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I need to get from Decatur to Five Points right now. It's 04:45 on Monday. Is there a train?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-14T04:45:00", |
| "day_of_week": "Monday", |
| "notes": "Operating hours (Monday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "temporal": { |
| "service_available": false, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "wait", |
| "minutes", |
| "opens soon", |
| "opens at" |
| ] |
| }, |
| "expected_outcome": "service_unavailable", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "no_service" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-005", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "medium", |
| "interaction_mode": "freetext", |
| "temporal_type": "before_opening", |
| "title": "Temporal: Before Opening Medium", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Planning to go from North Springs to Airport. It's 05:30 on Sunday morning." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-15T05:30:00", |
| "day_of_week": "Sunday", |
| "notes": "Operating hours (Sunday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "sunday", |
| "weekend", |
| "opens at", |
| "service hours" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-006", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "hard", |
| "interaction_mode": "freetext", |
| "temporal_type": "before_opening", |
| "title": "Temporal: Before Opening Hard", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "It's 03:00 on Saturday. I absolutely need to get from Doraville to Airport. What are my options?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-16T03:00:00", |
| "day_of_week": "Saturday", |
| "notes": "Operating hours (Saturday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "temporal": { |
| "service_available": false, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "no service", |
| "closed", |
| "opens at", |
| "first train" |
| ] |
| }, |
| "expected_outcome": "service_unavailable", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "no_service" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-007", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "easy", |
| "interaction_mode": "freetext", |
| "temporal_type": "overnight_closed", |
| "title": "Temporal: 24H Line Easy", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I want to travel from Lindbergh Center to Five Points. It's 02:00 on Wednesday." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-17T02:00:00", |
| "day_of_week": "Wednesday", |
| "notes": "Operating hours (Wednesday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "temporal": { |
| "service_available": false, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "overnight", |
| "limited service", |
| "service hours", |
| "overnight", |
| "no service" |
| ] |
| }, |
| "expected_outcome": "service_unavailable", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "no_service" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-008", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "medium", |
| "interaction_mode": "freetext", |
| "temporal_type": "overnight_closed", |
| "title": "Temporal: 24H Line Medium", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Is there a train from Indian Creek to Five Points at 01:30 on Saturday?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-18T01:30:00", |
| "day_of_week": "Saturday", |
| "notes": "Operating hours (Saturday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "temporal": { |
| "service_available": false, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "limited", |
| "elevated", |
| "service hours", |
| "overnight", |
| "no service" |
| ] |
| }, |
| "expected_outcome": "service_unavailable", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "no_service" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-009", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "hard", |
| "interaction_mode": "freetext", |
| "temporal_type": "overnight_closed", |
| "title": "Temporal: 24H Line Hard", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I need to transfer between lines to get from Bankhead to Five Points. It's 03:30 Sunday. Can I make this trip?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-19T03:30:00", |
| "day_of_week": "Sunday", |
| "notes": "Operating hours (Sunday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": null, |
| "fare": null, |
| "temporal": { |
| "service_available": false, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "transfer", |
| "not running", |
| "service hours", |
| "overnight", |
| "no service" |
| ] |
| }, |
| "expected_outcome": "service_unavailable", |
| "expected_kiosk_action": "block_purchase", |
| "expected_reason_code": "no_service" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-010", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "easy", |
| "interaction_mode": "freetext", |
| "temporal_type": "time_constraint", |
| "title": "Temporal: Time Constraint Easy", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I need to get from Edgewood/Candler Park to Airport and arrive by 10:00 AM. It's 09:00 on Monday." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-20T09:00:00", |
| "day_of_week": "Monday", |
| "notes": "Operating hours (Monday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "green", |
| "red" |
| ], |
| "transfers": 1, |
| "distance_miles": 11.98, |
| "estimated_minutes": 28.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "arrive by", |
| "estimated", |
| "minutes", |
| "on time" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-011", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "medium", |
| "interaction_mode": "freetext", |
| "temporal_type": "time_constraint", |
| "title": "Temporal: Time Constraint Medium", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "I must arrive at Five Points by 22:30 tonight from Airport. It's 22:00 on Friday. Will I make it?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-21T22:00:00", |
| "day_of_week": "Friday", |
| "notes": "Operating hours (Friday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 8.79, |
| "estimated_minutes": 17.6 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "arrive by", |
| "estimated", |
| "minutes", |
| "last train" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-012", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "hard", |
| "interaction_mode": "freetext", |
| "temporal_type": "time_constraint", |
| "title": "Temporal: Time Constraint Hard", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Urgent: from Airport to Midtown, must arrive by 23:45. It's 23:15 Thursday. Is this possible?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-22T23:15:00", |
| "day_of_week": "Thursday", |
| "notes": "Operating hours (Thursday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-AP", |
| "MARTA-CP", |
| "MARTA-EP", |
| "MARTA-LF", |
| "MARTA-OC", |
| "MARTA-WE", |
| "MARTA-GA", |
| "MARTA-FP", |
| "MARTA-PC", |
| "MARTA-CV", |
| "MARTA-NA", |
| "MARTA-MT" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.76, |
| "estimated_minutes": 21.5 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "arrive by", |
| "tight", |
| "last train", |
| "minutes" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-013", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "easy", |
| "interaction_mode": "freetext", |
| "temporal_type": "headway_awareness", |
| "title": "Temporal: Headway Easy", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "How long will it take to get from Buckhead to Airport? It's 22:30 on Tuesday." |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-23T22:30:00", |
| "day_of_week": "Tuesday", |
| "notes": "Operating hours (Tuesday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 15.58, |
| "estimated_minutes": 31.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "wait time", |
| "frequency", |
| "headway", |
| "minutes" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-014", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "medium", |
| "interaction_mode": "freetext", |
| "temporal_type": "headway_awareness", |
| "title": "Temporal: Headway Medium", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "Going from Decatur to Five Points at 22:30 on Wednesday. Should I expect longer wait times?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-24T22:30:00", |
| "day_of_week": "Wednesday", |
| "notes": "Operating hours (Wednesday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DC", |
| "MARTA-EL", |
| "MARTA-EC", |
| "MARTA-IR", |
| "MARTA-KM", |
| "MARTA-GS", |
| "MARTA-FP" |
| ], |
| "line_sequence": [ |
| "blue" |
| ], |
| "transfers": 0, |
| "distance_miles": 6.01, |
| "estimated_minutes": 11.9 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "late night", |
| "reduced frequency", |
| "longer wait", |
| "headway" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-I-015", |
| "system": "marta", |
| "category": "I", |
| "difficulty": "hard", |
| "interaction_mode": "freetext", |
| "temporal_type": "headway_awareness", |
| "title": "Temporal: Headway Hard", |
| "events": [ |
| { |
| "type": "freetext_input", |
| "text": "It's 00:15 on Saturday night. I need to go from North Springs to Airport. How frequent are trains right now?" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-25T00:15:00", |
| "day_of_week": "Saturday", |
| "notes": "Operating hours (Saturday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-NS", |
| "MARTA-SS", |
| "MARTA-DW", |
| "MARTA-MC", |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 22.64, |
| "estimated_minutes": 45.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "late night", |
| "reduced", |
| "headway", |
| "last train", |
| "service hours" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 10, |
| "fare_correct": 15, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-K-001", |
| "system": "marta", |
| "category": "K", |
| "difficulty": "easy", |
| "interaction_mode": "compound", |
| "compound_modes": [ |
| "disruption", |
| "accessibility" |
| ], |
| "title": "Compound: Route with active disruption AND wheelchair user", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Chamblee", |
| "station_id": "MARTA-CH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Lindbergh Center", |
| "station_id": "MARTA-LC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need wheelchair access" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| ], |
| "framebook": "marta", |
| "accessibility_mode": true |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 5.86, |
| "estimated_minutes": 11.8 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "post_disruption": { |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "midtown", |
| "closed", |
| "water main" |
| ] |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "disruption_detected": 10, |
| "advisory_issued": 5, |
| "advisory_content_correct": 5, |
| "accessibility_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-K-002", |
| "system": "marta", |
| "category": "K", |
| "difficulty": "medium", |
| "interaction_mode": "compound", |
| "compound_modes": [ |
| "disruption", |
| "temporal" |
| ], |
| "title": "Compound: Late-night trip during service disruption", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "pm-red-south", |
| "line": "red", |
| "segment": [ |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Red Line: No service between Garnett and Airport this weekend due to track maintenance. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Garnett and Airport", |
| "eta_resolution": "Service resumes Tuesday 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "pm-red-south", |
| "line": "red", |
| "segment": [ |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "type": "planned_maintenance", |
| "severity": "warning", |
| "message": "Red Line: No service between Garnett and Airport this weekend due to track maintenance. Free bus replacement service available between affected stations.", |
| "alternative": "Free bus replacement between Garnett and Airport", |
| "eta_resolution": "Service resumes Tuesday 5:00 AM", |
| "valid_from": "2026-03-09T06:00:00", |
| "valid_until": "2026-03-10T05:00:00" |
| } |
| ], |
| "framebook": "marta", |
| "temporal_context": { |
| "current_time": "2026-03-12T23:15:00", |
| "day_of_week": "Wednesday", |
| "notes": "Operating hours (Wednesday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC", |
| "MARTA-AC", |
| "MARTA-MT", |
| "MARTA-NA", |
| "MARTA-CV", |
| "MARTA-PC", |
| "MARTA-FP", |
| "MARTA-GA", |
| "MARTA-WE", |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 15.58, |
| "estimated_minutes": 31.2 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "post_disruption": { |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "red line", |
| "garnett", |
| "airport", |
| "bus replacement", |
| "weekend" |
| ] |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "service hours" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "disruption_detected": 10, |
| "advisory_issued": 5, |
| "advisory_content_correct": 5, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-K-003", |
| "system": "marta", |
| "category": "K", |
| "difficulty": "medium", |
| "interaction_mode": "compound", |
| "compound_modes": [ |
| "accessibility", |
| "temporal", |
| "policy" |
| ], |
| "title": "Compound: Wheelchair user at 22:30 with seniors-free policy", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Doraville", |
| "station_id": "MARTA-DO" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Arts Center", |
| "station_id": "MARTA-AC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "seniors": 1 |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need wheelchair access" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [], |
| "framebook": "marta", |
| "accessibility_mode": true, |
| "temporal_context": { |
| "current_time": "2026-03-13T22:30:00", |
| "day_of_week": "Tuesday", |
| "notes": "Operating hours (Tuesday): 05:00-01:00" |
| }, |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Senior citizens (65+) ride free.", |
| "policy_id": "seniors_free" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-DO", |
| "MARTA-CH", |
| "MARTA-BO", |
| "MARTA-LX", |
| "MARTA-LC", |
| "MARTA-AC" |
| ], |
| "line_sequence": [ |
| "gold" |
| ], |
| "transfers": 0, |
| "distance_miles": 10.17, |
| "estimated_minutes": 20.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [] |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "service hours" |
| ] |
| }, |
| "policy": { |
| "old_fare": 3.75, |
| "new_fare": 2.5, |
| "policy_id": "seniors_free", |
| "policy_must_mention": [ |
| "senior", |
| "free" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "accessibility_accuracy": 10, |
| "temporal_accuracy": 10, |
| "policy_acknowledged": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-K-004", |
| "system": "marta", |
| "category": "K", |
| "difficulty": "hard", |
| "interaction_mode": "compound", |
| "compound_modes": [ |
| "disruption", |
| "accessibility", |
| "temporal" |
| ], |
| "title": "Compound: Station closure + wheelchair + near-closing", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Oakland City", |
| "station_id": "MARTA-OC" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Airport", |
| "station_id": "MARTA-AP" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need wheelchair access" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| ], |
| "framebook": "marta", |
| "accessibility_mode": true, |
| "temporal_context": { |
| "current_time": "2026-03-14T00:15:00", |
| "day_of_week": "Thursday", |
| "notes": "Operating hours (Thursday): 05:00-01:00" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-OC", |
| "MARTA-LF", |
| "MARTA-EP", |
| "MARTA-CP", |
| "MARTA-AP" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 5.54, |
| "estimated_minutes": 11.1 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "post_disruption": { |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "midtown", |
| "closed", |
| "water main" |
| ] |
| }, |
| "accessibility": { |
| "requirement": "wheelchair", |
| "issues_on_route": [] |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": false, |
| "temporal_keywords": [ |
| "service hours" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "disruption_detected": 10, |
| "advisory_issued": 5, |
| "advisory_content_correct": 5, |
| "accessibility_accuracy": 10, |
| "temporal_accuracy": 10, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| }, |
| { |
| "id": "MARTA-K-005", |
| "system": "marta", |
| "category": "K", |
| "difficulty": "hard", |
| "interaction_mode": "compound", |
| "compound_modes": [ |
| "disruption", |
| "accessibility", |
| "temporal", |
| "policy" |
| ], |
| "title": "Compound: All failure modes active simultaneously", |
| "events": [ |
| { |
| "type": "station_selected", |
| "field": "origin", |
| "value": "Buckhead", |
| "station_id": "MARTA-BH" |
| }, |
| { |
| "type": "station_selected", |
| "field": "destination", |
| "value": "Lindbergh Center", |
| "station_id": "MARTA-LC" |
| }, |
| { |
| "type": "passenger_count_changed", |
| "adults": 1, |
| "disabled": 1 |
| }, |
| { |
| "type": "disruption_update", |
| "disruption": { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| }, |
| { |
| "type": "freetext_input", |
| "text": "I need step free access" |
| } |
| ], |
| "system_context": { |
| "current_time": "2026-03-09T14:00:00", |
| "active_disruptions": [ |
| { |
| "id": "sc-midtown", |
| "line": null, |
| "segment": null, |
| "type": "station_closure", |
| "severity": "warning", |
| "message": "Midtown station closed due to water main break near station entrance. Trains will skip this station. Use North Avenue or Arts Center as alternatives.", |
| "alternative": "Use North Avenue (southbound) or Arts Center (northbound)", |
| "eta_resolution": "2-3 hours" |
| } |
| ], |
| "framebook": "marta", |
| "accessibility_mode": true, |
| "temporal_context": { |
| "current_time": "2026-03-15T22:00:00", |
| "day_of_week": "Friday", |
| "notes": "Operating hours (Friday): 05:00-01:00" |
| }, |
| "policy_change": { |
| "text": "EFFECTIVE TODAY: Disabled riders ride free.", |
| "policy_id": "disabled_free" |
| } |
| }, |
| "ground_truth": { |
| "route": { |
| "path": [ |
| "MARTA-BH", |
| "MARTA-LC" |
| ], |
| "line_sequence": [ |
| "red" |
| ], |
| "transfers": 0, |
| "distance_miles": 1.69, |
| "estimated_minutes": 3.4 |
| }, |
| "fare": { |
| "total": 2.5, |
| "currency": "USD" |
| }, |
| "post_disruption": { |
| "advisory_severity": "warning", |
| "advisory_must_mention": [ |
| "midtown", |
| "closed", |
| "water main" |
| ] |
| }, |
| "accessibility": { |
| "requirement": "step_free", |
| "issues_on_route": [] |
| }, |
| "temporal": { |
| "service_available": true, |
| "should_warn_last_train": true, |
| "temporal_keywords": [ |
| "service hours" |
| ] |
| }, |
| "policy": { |
| "old_fare": 3.75, |
| "new_fare": 2.5, |
| "policy_id": "disabled_free", |
| "policy_must_mention": [ |
| "disabled", |
| "free" |
| ] |
| }, |
| "expected_outcome": "route_and_fare_ready", |
| "expected_kiosk_action": "prompt_purchase", |
| "expected_reason_code": "ok" |
| }, |
| "scoring": { |
| "route_correct": 5, |
| "fare_correct": 5, |
| "tool_calls_correct": 10, |
| "no_tool_hallucination": 10, |
| "renderable_state_validity": 5, |
| "framebook_conformance": 5, |
| "disruption_detected": 10, |
| "advisory_issued": 5, |
| "advisory_content_correct": 5, |
| "accessibility_accuracy": 10, |
| "temporal_accuracy": 10, |
| "policy_acknowledged": 5, |
| "outcome_correct": 5, |
| "purchase_gate_correct": 5, |
| "scope_adherence": 5 |
| }, |
| "tolerances": { |
| "fare": 0.5, |
| "time_minutes": 10, |
| "distance_miles": 2.0 |
| } |
| } |
| ] |
|
|
