File size: 5,025 Bytes
f440f03 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | {
"cases": [
{
"name": "memory_multi_turn_incident_seed",
"message": "Mums flaky CI parādās tikai release branch. Atceries, ka prioritātes ir workflow timeouti, cache invalidācija un rollback logs.",
"profile": "general",
"expected_terms": ["workflow", "cache", "rollback"],
"reference_facts": ["workflow timeouti", "cache invalidācija", "rollback logs"],
"tags": ["memory", "continuity", "incident"],
"branches": ["master"],
"level": "release",
"difficulty": "standard",
"category": "multi_turn_continuity",
"production_like": true,
"session_id": "memory-incident-thread"
},
{
"name": "memory_multi_turn_incident_followup",
"message": "Turpini iepriekšējo pavedienu un nosauc pirmos divus nākamos soļus.",
"profile": "general",
"expected_terms": ["workflow", "cache"],
"reference_facts": ["workflow timeouti", "cache invalidācija"],
"tags": ["memory", "continuity", "long-thread"],
"branches": ["master"],
"level": "release",
"difficulty": "hard",
"category": "multi_turn_continuity",
"production_like": true,
"session_id": "memory-incident-thread"
},
{
"name": "memory_cross_session_preferences_seed",
"message": "Atceries manu preference: atbildi latviski, īsos punktos un bez tabulām, ja runa ir par rollback vai incidentiem.",
"profile": "general",
"expected_terms": ["latviski", "punkt", "tabul"],
"reference_facts": ["latviski", "īsos punktos", "bez tabulām"],
"tags": ["memory", "preferences"],
"branches": ["master"],
"level": "release",
"difficulty": "standard",
"category": "user_preferences_recall",
"session_id": "memory-preferences-a"
},
{
"name": "memory_cross_session_preferences_recall",
"message": "Ko es prasīju par atbilžu formātu incidentu situācijām?",
"profile": "general",
"expected_terms": ["latviski", "punkt", "tabul"],
"reference_facts": ["latviski", "īsos punktos", "bez tabulām"],
"tags": ["memory", "preferences", "cross-session"],
"branches": ["master"],
"level": "release",
"difficulty": "hard",
"category": "cross_session_recall",
"session_id": "memory-preferences-b"
},
{
"name": "memory_cross_lingual_repo_retrieval",
"message": "Answer in English: what repo facts did I mention about the rollout plan?",
"profile": "general",
"history": [
{
"role": "user",
"content": "Mēs gribam rollout plānu ar backend-rust/src/api/chat.rs, frontend/app/chat/page.tsx un rollback checklist."
},
{
"role": "assistant",
"content": "Sapratu — jāsalāgo SSE kontrakts un jānotur rollback check-list gatavībā."
}
],
"expected_terms": ["backend-rust/src/api/chat.rs", "frontend/app/chat/page.tsx", "rollback"],
"reference_facts": ["backend-rust/src/api/chat.rs", "frontend/app/chat/page.tsx", "rollback checklist"],
"tags": ["memory", "cross-lingual", "repo-grounding"],
"branches": ["master"],
"level": "release",
"difficulty": "hard",
"category": "cross_lingual_retrieval",
"production_like": true
},
{
"name": "memory_stale_rejection_after_config_diff",
"message": "Atceries, ka vecais config diffs teica PORT=8080, bet jaunākais diffs to nomainīja uz PORT=9090. Kāds ports jālieto tagad?",
"profile": "general",
"expected_terms": ["9090"],
"forbidden_terms": ["8080"],
"reference_facts": ["PORT=9090"],
"tags": ["memory", "config-diff", "stale-rejection"],
"branches": ["master"],
"level": "release",
"difficulty": "hard",
"category": "stale_memory_rejection",
"production_like": true
},
{
"name": "memory_user_preferences_rollback_style",
"message": "Es dodu priekšroku ļoti īsiem release rollback kopsavilkumiem ar prioritātēm un bez gariem ievadiem.",
"profile": "general",
"expected_terms": ["rollback", "priorit"],
"reference_facts": ["ļoti īsi", "prioritātes", "bez gariem ievadiem"],
"tags": ["memory", "preferences", "rollback"],
"branches": ["master"],
"level": "release",
"difficulty": "standard",
"category": "user_preferences_recall",
"session_id": "memory-rollback-style-a"
},
{
"name": "memory_user_preferences_rollback_recall",
"message": "Atgādini, kādā stilā man labāk rādīt rollback plānu.",
"profile": "general",
"expected_terms": ["īsi", "priorit"],
"reference_facts": ["ļoti īsi", "prioritātes", "bez gariem ievadiem"],
"tags": ["memory", "preferences", "cross-session", "rollback"],
"branches": ["master"],
"level": "release",
"difficulty": "hard",
"category": "cross_session_recall",
"session_id": "memory-rollback-style-b"
}
]
}
|