rohitsar567 Claude Opus 4.7 (1M context) commited on
Commit
64f55e3
·
1 Parent(s): 21ffc50

fix(indic): KI-110 — repair Hindi pipeline raising on every turn

Browse files

Live re-smoke verification (2026-05-15) caught every Hindi/Devanagari turn
in Scenario S2 surfacing `brain_used=error_fallback` from KI-106's catch-all
exception handler. Root cause: post-KI-099 the translator's try/except only
catches `asyncio.TimeoutError` — but the KI-099 split httpx.Timeout(connect=2,
read=20, write=2, pool=2) means httpx can raise other exception types that
are NOT subclasses of asyncio.TimeoutError:

- httpx.ReadTimeout (inner 20s read budget; races the outer wait_for(20s))
- httpx.ConnectError / ConnectTimeout (network / DNS / TLS failures)
- httpx.HTTPStatusError (Sarvam 429 rate-limit on Indic-heavy load, or 5xx
server hiccups, surfaced by resp.raise_for_status())
- KeyError ("choices") on malformed Sarvam payload
- ValueError on malformed JSON / dataclass construction

Any of these would propagate up through translate_to_english /
translate_to_indic. While orchestrator's wrapping `except Exception` catches
the inbound call, translation_check.check_back_translation chains another
translate_to_english that — combined with sibling KI-108/109 reorganisation
of the indic cascade — could still surface as the user-facing error.

Fix: translator contract is now "NEVER raise on a Sarvam failure". Both
translate_to_english and translate_to_indic catch the explicit tuple of
Sarvam-originated exception types (_TRANSLATOR_FAILURE_TYPES) and passthrough
the original text on any failure. Logged with type-name so Sarvam health
remains observable from logs.

We deliberately do NOT catch asyncio.CancelledError (would break task
cancellation), and we do NOT use bare `except Exception` (would hide real
coding bugs in the translator path itself).

Files touched:
- backend/translator.py: explicit failure-type tuple, broadened catch
- tests/test_translator.py: NEW — 11 regression tests covering every
Sarvam failure mode (httpx.ReadTimeout, ConnectError, HTTPStatusError,
KeyError, asyncio.TimeoutError, malformed payload, empty input) and a
CRITICAL test that asyncio.CancelledError still propagates

Test results: 86 passed (75 existing + 11 new), 29 subtests passed, 0 failures.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Files changed (2) hide show
  1. backend/translator.py +45 -6
  2. tests/test_translator.py +139 -0
backend/translator.py CHANGED
@@ -20,6 +20,8 @@ from __future__ import annotations
20
  import asyncio
21
  import logging
22
 
 
 
23
  from backend.providers.base import ChatMessage
24
  from backend.providers.sarvam_llm import SarvamLLM
25
 
@@ -31,6 +33,35 @@ from backend.providers.sarvam_llm import SarvamLLM
31
  # something is genuinely wedged.
32
  _SARVAM_CALL_TIMEOUT = 20.0
33
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
  _log = logging.getLogger(__name__)
35
 
36
 
@@ -82,10 +113,14 @@ async def translate_to_english(text: str, sarvam: SarvamLLM | None = None) -> st
82
  ),
83
  timeout=_SARVAM_CALL_TIMEOUT,
84
  )
85
- except asyncio.TimeoutError:
 
 
 
 
86
  _log.warning(
87
- "sarvam translate_to_english wait_for timed out after %.1fs — passthrough",
88
- _SARVAM_CALL_TIMEOUT,
89
  )
90
  return text
91
  out = res.text.strip()
@@ -123,10 +158,14 @@ async def translate_to_indic(
123
  ),
124
  timeout=_SARVAM_CALL_TIMEOUT,
125
  )
126
- except asyncio.TimeoutError:
 
 
 
 
127
  _log.warning(
128
- "sarvam translate_to_indic wait_for timed out after %.1fs — passthrough",
129
- _SARVAM_CALL_TIMEOUT,
130
  )
131
  return english
132
  out = res.text.strip()
 
20
  import asyncio
21
  import logging
22
 
23
+ import httpx
24
+
25
  from backend.providers.base import ChatMessage
26
  from backend.providers.sarvam_llm import SarvamLLM
27
 
 
33
  # something is genuinely wedged.
34
  _SARVAM_CALL_TIMEOUT = 20.0
35
 
36
+
37
+ # KI-110 (2026-05-15) — translator contract: NEVER raise on a Sarvam failure.
38
+ # Callers (orchestrator inbound translate, cascade outbound translate,
39
+ # translation_check back-translate) all degrade gracefully when passthrough
40
+ # text is returned, but any exception escaping translate_to_english /
41
+ # translate_to_indic is a P0: live re-smoke verified that EVERY Hindi /
42
+ # Devanagari turn surfaced `brain_used=error_fallback` from KI-106's catch-all
43
+ # because the translator could raise non-TimeoutError exceptions and the
44
+ # catch only handled `asyncio.TimeoutError`.
45
+ #
46
+ # Specifically, post-KI-099 the inner httpx.Timeout(read=20.0) races with the
47
+ # outer asyncio.wait_for(timeout=20.0). When httpx fires first it raises
48
+ # `httpx.ReadTimeout` (NOT a subclass of asyncio.TimeoutError); on Sarvam 4xx
49
+ # or 5xx (rate-limit on Indic-heavy load, auth issues, server hiccups) it
50
+ # raises `httpx.HTTPStatusError`; on malformed payload it raises KeyError on
51
+ # `payload["choices"][0]`. None were caught.
52
+ #
53
+ # Fix: catch every Sarvam-originated exception type explicitly. We do NOT
54
+ # catch `asyncio.CancelledError` (would break task cancellation), and we do
55
+ # NOT use a bare `except Exception` (would hide real coding bugs).
56
+ _TRANSLATOR_FAILURE_TYPES: tuple[type[BaseException], ...] = (
57
+ asyncio.TimeoutError,
58
+ httpx.TimeoutException, # ConnectTimeout / ReadTimeout / WriteTimeout / PoolTimeout
59
+ httpx.HTTPStatusError, # 4xx/5xx from Sarvam
60
+ httpx.RequestError, # network / DNS / SSL / connect errors
61
+ KeyError, # malformed Sarvam payload (missing 'choices')
62
+ ValueError, # malformed JSON / dataclass construction
63
+ )
64
+
65
  _log = logging.getLogger(__name__)
66
 
67
 
 
113
  ),
114
  timeout=_SARVAM_CALL_TIMEOUT,
115
  )
116
+ except _TRANSLATOR_FAILURE_TYPES as e:
117
+ # KI-110 — passthrough on ANY Sarvam failure (timeout, http error,
118
+ # network error, malformed payload). The translator contract is
119
+ # "never raise" so callers don't see this as a chat-killing
120
+ # exception. Log with type-name so we can still observe Sarvam health.
121
  _log.warning(
122
+ "sarvam translate_to_english passthrough (%s): %s",
123
+ type(e).__name__, str(e)[:200],
124
  )
125
  return text
126
  out = res.text.strip()
 
158
  ),
159
  timeout=_SARVAM_CALL_TIMEOUT,
160
  )
161
+ except _TRANSLATOR_FAILURE_TYPES as e:
162
+ # KI-110 — see translate_to_english for full rationale. Passthrough
163
+ # the English on any Sarvam failure; orchestrator's indic cascade
164
+ # treats English-back == English-input as "no cascade translation"
165
+ # and serves the English reply unchanged.
166
  _log.warning(
167
+ "sarvam translate_to_indic passthrough (%s): %s",
168
+ type(e).__name__, str(e)[:200],
169
  )
170
  return english
171
  out = res.text.strip()
tests/test_translator.py ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Regression tests for backend.translator — KI-110.
2
+
3
+ Locks in the translator contract: NEVER raise on a Sarvam failure. Live
4
+ re-smoke (2026-05-15) caught every Hindi turn in Scenario S2 surfacing
5
+ `brain_used=error_fallback` because the post-KI-099 try/except only caught
6
+ `asyncio.TimeoutError` — leaving httpx.ReadTimeout, httpx.HTTPStatusError,
7
+ httpx.RequestError, and KeyError (malformed Sarvam payload) to propagate
8
+ through to KI-106's catch-all in main.py.
9
+
10
+ These tests assert: every Sarvam failure mode returns the original text
11
+ (passthrough), not raises. Mirrors the existing project convention of
12
+ asyncio.run inside unittest.TestCase (no pytest-asyncio dependency).
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import asyncio
17
+ import os
18
+ import sys
19
+ import unittest
20
+ from unittest.mock import AsyncMock
21
+
22
+ import httpx
23
+
24
+ # Ensure SARVAM_API_KEY is set so SarvamLLM constructor doesn't reject
25
+ os.environ.setdefault("SARVAM_API_KEY", "test-key-stub")
26
+
27
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
28
+
29
+ from backend.providers.base import LLMResult # noqa: E402
30
+ from backend.providers.sarvam_llm import SarvamLLM # noqa: E402
31
+ from backend.translator import translate_to_english, translate_to_indic # noqa: E402
32
+
33
+
34
+ HINDI_QUERY = "मुझे स्वास्थ्य बीमा चाहिए"
35
+ ENGLISH_REPLY = "Yes, HDFC ERGO Optima Secure covers Ayurveda treatment."
36
+
37
+
38
+ def _make_request() -> httpx.Request:
39
+ """httpx exceptions need a Request; construct a minimal one."""
40
+ return httpx.Request("POST", "https://api.sarvam.ai/v1/chat/completions")
41
+
42
+
43
+ def _stub_sarvam(side_effect=None, return_value=None) -> SarvamLLM:
44
+ fake = SarvamLLM(api_key="stub")
45
+ if side_effect is not None:
46
+ fake.chat = AsyncMock(side_effect=side_effect) # type: ignore[method-assign]
47
+ else:
48
+ fake.chat = AsyncMock(return_value=return_value) # type: ignore[method-assign]
49
+ return fake
50
+
51
+
52
+ class TranslateToEnglishTests(unittest.TestCase):
53
+ def test_happy_path(self):
54
+ fake = _stub_sarvam(return_value=LLMResult(
55
+ text="I want health insurance",
56
+ model="sarvam-m",
57
+ ))
58
+ out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
59
+ self.assertEqual(out, "I want health insurance")
60
+
61
+ def test_passthrough_on_asyncio_timeout(self):
62
+ """KI-099 baseline: outer wait_for fires → passthrough original Hindi."""
63
+ fake = _stub_sarvam(side_effect=asyncio.TimeoutError())
64
+ out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
65
+ self.assertEqual(out, HINDI_QUERY)
66
+
67
+ def test_passthrough_on_httpx_read_timeout(self):
68
+ """KI-110 P0: httpx.ReadTimeout (NOT subclass of asyncio.TimeoutError) — passthrough."""
69
+ fake = _stub_sarvam(side_effect=httpx.ReadTimeout(
70
+ "Read timeout", request=_make_request(),
71
+ ))
72
+ out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
73
+ self.assertEqual(out, HINDI_QUERY)
74
+
75
+ def test_passthrough_on_httpx_connect_error(self):
76
+ """KI-110: Sarvam unreachable — passthrough."""
77
+ fake = _stub_sarvam(side_effect=httpx.ConnectError(
78
+ "Connection refused", request=_make_request(),
79
+ ))
80
+ out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
81
+ self.assertEqual(out, HINDI_QUERY)
82
+
83
+ def test_passthrough_on_http_status_error(self):
84
+ """KI-110: Sarvam 429 (rate-limit on Indic-heavy load) — passthrough."""
85
+ req = _make_request()
86
+ resp = httpx.Response(status_code=429, request=req)
87
+ fake = _stub_sarvam(side_effect=httpx.HTTPStatusError(
88
+ "Too Many Requests", request=req, response=resp,
89
+ ))
90
+ out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
91
+ self.assertEqual(out, HINDI_QUERY)
92
+
93
+ def test_passthrough_on_malformed_payload(self):
94
+ """KI-110: Sarvam returns malformed payload (no 'choices' key) — passthrough."""
95
+ fake = _stub_sarvam(side_effect=KeyError("choices"))
96
+ out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
97
+ self.assertEqual(out, HINDI_QUERY)
98
+
99
+ def test_empty_input_no_sarvam_call(self):
100
+ """Empty input: returns immediately, no Sarvam call."""
101
+ self.assertEqual(asyncio.run(translate_to_english("")), "")
102
+ self.assertEqual(asyncio.run(translate_to_english(" ")), " ")
103
+
104
+ def test_cancelled_error_propagates(self):
105
+ """CRITICAL: asyncio.CancelledError MUST propagate — never swallow it,
106
+ otherwise outer wait_for / task cancellation breaks."""
107
+ fake = _stub_sarvam(side_effect=asyncio.CancelledError())
108
+ with self.assertRaises(asyncio.CancelledError):
109
+ asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
110
+
111
+
112
+ class TranslateToIndicTests(unittest.TestCase):
113
+ def test_happy_path(self):
114
+ fake = _stub_sarvam(return_value=LLMResult(
115
+ text="Haan, HDFC ERGO Optima Secure mein Ayurveda cover hai.",
116
+ model="sarvam-m",
117
+ ))
118
+ out = asyncio.run(translate_to_indic(ENGLISH_REPLY, sarvam=fake))
119
+ self.assertIn("HDFC", out)
120
+
121
+ def test_passthrough_on_httpx_read_timeout(self):
122
+ fake = _stub_sarvam(side_effect=httpx.ReadTimeout(
123
+ "Read timeout", request=_make_request(),
124
+ ))
125
+ out = asyncio.run(translate_to_indic(ENGLISH_REPLY, sarvam=fake))
126
+ self.assertEqual(out, ENGLISH_REPLY)
127
+
128
+ def test_passthrough_on_http_status_error(self):
129
+ req = _make_request()
130
+ resp = httpx.Response(status_code=500, request=req)
131
+ fake = _stub_sarvam(side_effect=httpx.HTTPStatusError(
132
+ "Server Error", request=req, response=resp,
133
+ ))
134
+ out = asyncio.run(translate_to_indic(ENGLISH_REPLY, sarvam=fake))
135
+ self.assertEqual(out, ENGLISH_REPLY)
136
+
137
+
138
+ if __name__ == "__main__":
139
+ unittest.main()