Sync from GitHub 73909d4496990ebe682e14cac6037513d7188032

tests/test_llm_fact_eval.py

@@ -223,8 +223,8 @@ CASES = build_cases() if os.getenv("RUN_LLM_EVAL") else []
 def make_chain():
     from src.rag.agent_chain import ExecutiveAgentChain
 
-    def _factory(lang: str):
-        return ExecutiveAgentChain(language=lang, session_id=f"eval_{lang}")
+    def _factory(lang: str, attempt: int = 0):
+        return ExecutiveAgentChain(language=lang, session_id=f"eval_{lang}_{attempt}")
 
     return _factory
 
@@ -233,12 +233,16 @@ def make_chain():
 # catches gross regressions such as switching back to a reasoning model).
 MAX_TURN_SECONDS = 25.0
 
+# LLM answers are sampled, so a single case can flake even though the facts are
+# in the prompt (observed on main: a different case failed on each smoke run).
+# One retry with a fresh chain/session filters sampling noise; a real
+# regression (wrong facts, cross-contamination, latency) still fails twice.
+FLAKE_RETRIES = 1
 
-@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES])
-def test_fact_eval(case, make_chain):
+
+def _assert_case(case, chain):
     from time import perf_counter
 
-    chain = make_chain(case["lang"])
     turn_start = perf_counter()
     result = chain.query(case["query"])
     elapsed = perf_counter() - turn_start
@@ -266,3 +270,14 @@ def test_fact_eval(case, make_chain):
             f"\nFORBIDDEN token found (cross-programme contamination): {forbidden}"
             f"\nAnswer:   {answer[:600]}"
         )
+
+
+@pytest.mark.parametrize("case", CASES, ids=[c["id"] for c in CASES])
+def test_fact_eval(case, make_chain):
+    for attempt in range(FLAKE_RETRIES + 1):
+        try:
+            _assert_case(case, make_chain(case["lang"], attempt))
+            return
+        except AssertionError:
+            if attempt == FLAKE_RETRIES:
+                raise

tests/test_uat_llm_judge.py

@@ -611,14 +611,56 @@ def test_uat_mixed_language_case_expects_english_clarification():
     assert cases["TC-EDGE-05"].mixed_language_clarification_expected is True
 
 
+def test_tuition_deadline_status_enrichment_is_deterministic():
+    tuition = {
+        "first_deadline": {"deadline": "2026-03-31", "fee": 80000},
+        "final_deadline": {"deadline": "2026-06-30", "fee": 85000},
+    }
+
+    enriched = _tuition_with_deadline_status(tuition, date(2026, 6, 30))
+
+    assert enriched["first_deadline"]["deadline_status"] == "passed"
+    assert enriched["final_deadline"]["deadline_status"] == "due_today"
+    assert enriched["applicable_today"] == {
+        "deadline_type": "final_deadline",
+        "deadline": "2026-06-30",
+        "deadline_status": "due_today",
+        "fee": 85000,
+    }
+
+    all_passed = _tuition_with_deadline_status(tuition, date(2026, 7, 1))
+    assert all_passed["applicable_today"]["deadline_type"] == "final_deadline"
+
+    all_future = _tuition_with_deadline_status(tuition, date(2026, 1, 1))
+    assert all_future["applicable_today"]["deadline_type"] == "first_deadline"
+    assert all_future["applicable_today"]["fee"] == 80000
+
+
 def test_hard_facts_include_deadline_status_and_applicable_fee():
     facts = _hard_facts()
 
-    assert facts["reference_date"] == "2026-06-30"
-    assert facts["programmes"]["iemba"]["tuition_chf"]["first_deadline"]["deadline_status"] == "passed"
-    assert facts["programmes"]["iemba"]["tuition_chf"]["final_deadline"]["deadline_status"] == "due_today"
-    assert facts["programmes"]["iemba"]["tuition_chf"]["applicable_today"]["fee"] == 85000
-    assert facts["programmes"]["emba_x"]["tuition_chf"]["applicable_today"]["fee"] == 99000
+    # generated_at moves with every daily facts refresh, so assert consistency
+    # against the file instead of hardcoding a snapshot date.
+    assert facts["reference_date"] == (facts["generated_at"] or "")[:10]
+    reference_date = date.fromisoformat(facts["reference_date"])
+
+    for programme in facts["programmes"].values():
+        tuition = programme["tuition_chf"]
+        for key in ("first_deadline", "final_deadline"):
+            entry = tuition[key]
+            assert entry["deadline_status"] == _deadline_status(
+                entry.get("deadline"), reference_date
+            )
+
+        applicable = tuition["applicable_today"]
+        chosen = tuition[applicable["deadline_type"]]
+        assert applicable["fee"] == chosen["fee"]
+        assert applicable["deadline"] == chosen["deadline"]
+        assert applicable["deadline_status"] == chosen["deadline_status"]
+        if tuition["first_deadline"]["deadline_status"] in {"future", "due_today"}:
+            assert applicable["deadline_type"] == "first_deadline"
+        else:
+            assert applicable["deadline_type"] == "final_deadline"
 
 
 def test_uat_transcript_starts_with_bot_greeting(monkeypatch):