Spaces:
Sleeping
fix(indic): KI-110 — repair Hindi pipeline raising on every turn
Browse filesLive re-smoke verification (2026-05-15) caught every Hindi/Devanagari turn
in Scenario S2 surfacing `brain_used=error_fallback` from KI-106's catch-all
exception handler. Root cause: post-KI-099 the translator's try/except only
catches `asyncio.TimeoutError` — but the KI-099 split httpx.Timeout(connect=2,
read=20, write=2, pool=2) means httpx can raise other exception types that
are NOT subclasses of asyncio.TimeoutError:
- httpx.ReadTimeout (inner 20s read budget; races the outer wait_for(20s))
- httpx.ConnectError / ConnectTimeout (network / DNS / TLS failures)
- httpx.HTTPStatusError (Sarvam 429 rate-limit on Indic-heavy load, or 5xx
server hiccups, surfaced by resp.raise_for_status())
- KeyError ("choices") on malformed Sarvam payload
- ValueError on malformed JSON / dataclass construction
Any of these would propagate up through translate_to_english /
translate_to_indic. While orchestrator's wrapping `except Exception` catches
the inbound call, translation_check.check_back_translation chains another
translate_to_english that — combined with sibling KI-108/109 reorganisation
of the indic cascade — could still surface as the user-facing error.
Fix: translator contract is now "NEVER raise on a Sarvam failure". Both
translate_to_english and translate_to_indic catch the explicit tuple of
Sarvam-originated exception types (_TRANSLATOR_FAILURE_TYPES) and passthrough
the original text on any failure. Logged with type-name so Sarvam health
remains observable from logs.
We deliberately do NOT catch asyncio.CancelledError (would break task
cancellation), and we do NOT use bare `except Exception` (would hide real
coding bugs in the translator path itself).
Files touched:
- backend/translator.py: explicit failure-type tuple, broadened catch
- tests/test_translator.py: NEW — 11 regression tests covering every
Sarvam failure mode (httpx.ReadTimeout, ConnectError, HTTPStatusError,
KeyError, asyncio.TimeoutError, malformed payload, empty input) and a
CRITICAL test that asyncio.CancelledError still propagates
Test results: 86 passed (75 existing + 11 new), 29 subtests passed, 0 failures.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
- backend/translator.py +45 -6
- tests/test_translator.py +139 -0
|
@@ -20,6 +20,8 @@ from __future__ import annotations
|
|
| 20 |
import asyncio
|
| 21 |
import logging
|
| 22 |
|
|
|
|
|
|
|
| 23 |
from backend.providers.base import ChatMessage
|
| 24 |
from backend.providers.sarvam_llm import SarvamLLM
|
| 25 |
|
|
@@ -31,6 +33,35 @@ from backend.providers.sarvam_llm import SarvamLLM
|
|
| 31 |
# something is genuinely wedged.
|
| 32 |
_SARVAM_CALL_TIMEOUT = 20.0
|
| 33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
_log = logging.getLogger(__name__)
|
| 35 |
|
| 36 |
|
|
@@ -82,10 +113,14 @@ async def translate_to_english(text: str, sarvam: SarvamLLM | None = None) -> st
|
|
| 82 |
),
|
| 83 |
timeout=_SARVAM_CALL_TIMEOUT,
|
| 84 |
)
|
| 85 |
-
except
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
_log.warning(
|
| 87 |
-
"sarvam translate_to_english
|
| 88 |
-
|
| 89 |
)
|
| 90 |
return text
|
| 91 |
out = res.text.strip()
|
|
@@ -123,10 +158,14 @@ async def translate_to_indic(
|
|
| 123 |
),
|
| 124 |
timeout=_SARVAM_CALL_TIMEOUT,
|
| 125 |
)
|
| 126 |
-
except
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
_log.warning(
|
| 128 |
-
"sarvam translate_to_indic
|
| 129 |
-
|
| 130 |
)
|
| 131 |
return english
|
| 132 |
out = res.text.strip()
|
|
|
|
| 20 |
import asyncio
|
| 21 |
import logging
|
| 22 |
|
| 23 |
+
import httpx
|
| 24 |
+
|
| 25 |
from backend.providers.base import ChatMessage
|
| 26 |
from backend.providers.sarvam_llm import SarvamLLM
|
| 27 |
|
|
|
|
| 33 |
# something is genuinely wedged.
|
| 34 |
_SARVAM_CALL_TIMEOUT = 20.0
|
| 35 |
|
| 36 |
+
|
| 37 |
+
# KI-110 (2026-05-15) — translator contract: NEVER raise on a Sarvam failure.
|
| 38 |
+
# Callers (orchestrator inbound translate, cascade outbound translate,
|
| 39 |
+
# translation_check back-translate) all degrade gracefully when passthrough
|
| 40 |
+
# text is returned, but any exception escaping translate_to_english /
|
| 41 |
+
# translate_to_indic is a P0: live re-smoke verified that EVERY Hindi /
|
| 42 |
+
# Devanagari turn surfaced `brain_used=error_fallback` from KI-106's catch-all
|
| 43 |
+
# because the translator could raise non-TimeoutError exceptions and the
|
| 44 |
+
# catch only handled `asyncio.TimeoutError`.
|
| 45 |
+
#
|
| 46 |
+
# Specifically, post-KI-099 the inner httpx.Timeout(read=20.0) races with the
|
| 47 |
+
# outer asyncio.wait_for(timeout=20.0). When httpx fires first it raises
|
| 48 |
+
# `httpx.ReadTimeout` (NOT a subclass of asyncio.TimeoutError); on Sarvam 4xx
|
| 49 |
+
# or 5xx (rate-limit on Indic-heavy load, auth issues, server hiccups) it
|
| 50 |
+
# raises `httpx.HTTPStatusError`; on malformed payload it raises KeyError on
|
| 51 |
+
# `payload["choices"][0]`. None were caught.
|
| 52 |
+
#
|
| 53 |
+
# Fix: catch every Sarvam-originated exception type explicitly. We do NOT
|
| 54 |
+
# catch `asyncio.CancelledError` (would break task cancellation), and we do
|
| 55 |
+
# NOT use a bare `except Exception` (would hide real coding bugs).
|
| 56 |
+
_TRANSLATOR_FAILURE_TYPES: tuple[type[BaseException], ...] = (
|
| 57 |
+
asyncio.TimeoutError,
|
| 58 |
+
httpx.TimeoutException, # ConnectTimeout / ReadTimeout / WriteTimeout / PoolTimeout
|
| 59 |
+
httpx.HTTPStatusError, # 4xx/5xx from Sarvam
|
| 60 |
+
httpx.RequestError, # network / DNS / SSL / connect errors
|
| 61 |
+
KeyError, # malformed Sarvam payload (missing 'choices')
|
| 62 |
+
ValueError, # malformed JSON / dataclass construction
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
_log = logging.getLogger(__name__)
|
| 66 |
|
| 67 |
|
|
|
|
| 113 |
),
|
| 114 |
timeout=_SARVAM_CALL_TIMEOUT,
|
| 115 |
)
|
| 116 |
+
except _TRANSLATOR_FAILURE_TYPES as e:
|
| 117 |
+
# KI-110 — passthrough on ANY Sarvam failure (timeout, http error,
|
| 118 |
+
# network error, malformed payload). The translator contract is
|
| 119 |
+
# "never raise" so callers don't see this as a chat-killing
|
| 120 |
+
# exception. Log with type-name so we can still observe Sarvam health.
|
| 121 |
_log.warning(
|
| 122 |
+
"sarvam translate_to_english passthrough (%s): %s",
|
| 123 |
+
type(e).__name__, str(e)[:200],
|
| 124 |
)
|
| 125 |
return text
|
| 126 |
out = res.text.strip()
|
|
|
|
| 158 |
),
|
| 159 |
timeout=_SARVAM_CALL_TIMEOUT,
|
| 160 |
)
|
| 161 |
+
except _TRANSLATOR_FAILURE_TYPES as e:
|
| 162 |
+
# KI-110 — see translate_to_english for full rationale. Passthrough
|
| 163 |
+
# the English on any Sarvam failure; orchestrator's indic cascade
|
| 164 |
+
# treats English-back == English-input as "no cascade translation"
|
| 165 |
+
# and serves the English reply unchanged.
|
| 166 |
_log.warning(
|
| 167 |
+
"sarvam translate_to_indic passthrough (%s): %s",
|
| 168 |
+
type(e).__name__, str(e)[:200],
|
| 169 |
)
|
| 170 |
return english
|
| 171 |
out = res.text.strip()
|
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Regression tests for backend.translator — KI-110.
|
| 2 |
+
|
| 3 |
+
Locks in the translator contract: NEVER raise on a Sarvam failure. Live
|
| 4 |
+
re-smoke (2026-05-15) caught every Hindi turn in Scenario S2 surfacing
|
| 5 |
+
`brain_used=error_fallback` because the post-KI-099 try/except only caught
|
| 6 |
+
`asyncio.TimeoutError` — leaving httpx.ReadTimeout, httpx.HTTPStatusError,
|
| 7 |
+
httpx.RequestError, and KeyError (malformed Sarvam payload) to propagate
|
| 8 |
+
through to KI-106's catch-all in main.py.
|
| 9 |
+
|
| 10 |
+
These tests assert: every Sarvam failure mode returns the original text
|
| 11 |
+
(passthrough), not raises. Mirrors the existing project convention of
|
| 12 |
+
asyncio.run inside unittest.TestCase (no pytest-asyncio dependency).
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import asyncio
|
| 17 |
+
import os
|
| 18 |
+
import sys
|
| 19 |
+
import unittest
|
| 20 |
+
from unittest.mock import AsyncMock
|
| 21 |
+
|
| 22 |
+
import httpx
|
| 23 |
+
|
| 24 |
+
# Ensure SARVAM_API_KEY is set so SarvamLLM constructor doesn't reject
|
| 25 |
+
os.environ.setdefault("SARVAM_API_KEY", "test-key-stub")
|
| 26 |
+
|
| 27 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 28 |
+
|
| 29 |
+
from backend.providers.base import LLMResult # noqa: E402
|
| 30 |
+
from backend.providers.sarvam_llm import SarvamLLM # noqa: E402
|
| 31 |
+
from backend.translator import translate_to_english, translate_to_indic # noqa: E402
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
HINDI_QUERY = "मुझे स्वास्थ्य बीमा चाहिए"
|
| 35 |
+
ENGLISH_REPLY = "Yes, HDFC ERGO Optima Secure covers Ayurveda treatment."
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _make_request() -> httpx.Request:
|
| 39 |
+
"""httpx exceptions need a Request; construct a minimal one."""
|
| 40 |
+
return httpx.Request("POST", "https://api.sarvam.ai/v1/chat/completions")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _stub_sarvam(side_effect=None, return_value=None) -> SarvamLLM:
|
| 44 |
+
fake = SarvamLLM(api_key="stub")
|
| 45 |
+
if side_effect is not None:
|
| 46 |
+
fake.chat = AsyncMock(side_effect=side_effect) # type: ignore[method-assign]
|
| 47 |
+
else:
|
| 48 |
+
fake.chat = AsyncMock(return_value=return_value) # type: ignore[method-assign]
|
| 49 |
+
return fake
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class TranslateToEnglishTests(unittest.TestCase):
|
| 53 |
+
def test_happy_path(self):
|
| 54 |
+
fake = _stub_sarvam(return_value=LLMResult(
|
| 55 |
+
text="I want health insurance",
|
| 56 |
+
model="sarvam-m",
|
| 57 |
+
))
|
| 58 |
+
out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
|
| 59 |
+
self.assertEqual(out, "I want health insurance")
|
| 60 |
+
|
| 61 |
+
def test_passthrough_on_asyncio_timeout(self):
|
| 62 |
+
"""KI-099 baseline: outer wait_for fires → passthrough original Hindi."""
|
| 63 |
+
fake = _stub_sarvam(side_effect=asyncio.TimeoutError())
|
| 64 |
+
out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
|
| 65 |
+
self.assertEqual(out, HINDI_QUERY)
|
| 66 |
+
|
| 67 |
+
def test_passthrough_on_httpx_read_timeout(self):
|
| 68 |
+
"""KI-110 P0: httpx.ReadTimeout (NOT subclass of asyncio.TimeoutError) — passthrough."""
|
| 69 |
+
fake = _stub_sarvam(side_effect=httpx.ReadTimeout(
|
| 70 |
+
"Read timeout", request=_make_request(),
|
| 71 |
+
))
|
| 72 |
+
out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
|
| 73 |
+
self.assertEqual(out, HINDI_QUERY)
|
| 74 |
+
|
| 75 |
+
def test_passthrough_on_httpx_connect_error(self):
|
| 76 |
+
"""KI-110: Sarvam unreachable — passthrough."""
|
| 77 |
+
fake = _stub_sarvam(side_effect=httpx.ConnectError(
|
| 78 |
+
"Connection refused", request=_make_request(),
|
| 79 |
+
))
|
| 80 |
+
out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
|
| 81 |
+
self.assertEqual(out, HINDI_QUERY)
|
| 82 |
+
|
| 83 |
+
def test_passthrough_on_http_status_error(self):
|
| 84 |
+
"""KI-110: Sarvam 429 (rate-limit on Indic-heavy load) — passthrough."""
|
| 85 |
+
req = _make_request()
|
| 86 |
+
resp = httpx.Response(status_code=429, request=req)
|
| 87 |
+
fake = _stub_sarvam(side_effect=httpx.HTTPStatusError(
|
| 88 |
+
"Too Many Requests", request=req, response=resp,
|
| 89 |
+
))
|
| 90 |
+
out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
|
| 91 |
+
self.assertEqual(out, HINDI_QUERY)
|
| 92 |
+
|
| 93 |
+
def test_passthrough_on_malformed_payload(self):
|
| 94 |
+
"""KI-110: Sarvam returns malformed payload (no 'choices' key) — passthrough."""
|
| 95 |
+
fake = _stub_sarvam(side_effect=KeyError("choices"))
|
| 96 |
+
out = asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
|
| 97 |
+
self.assertEqual(out, HINDI_QUERY)
|
| 98 |
+
|
| 99 |
+
def test_empty_input_no_sarvam_call(self):
|
| 100 |
+
"""Empty input: returns immediately, no Sarvam call."""
|
| 101 |
+
self.assertEqual(asyncio.run(translate_to_english("")), "")
|
| 102 |
+
self.assertEqual(asyncio.run(translate_to_english(" ")), " ")
|
| 103 |
+
|
| 104 |
+
def test_cancelled_error_propagates(self):
|
| 105 |
+
"""CRITICAL: asyncio.CancelledError MUST propagate — never swallow it,
|
| 106 |
+
otherwise outer wait_for / task cancellation breaks."""
|
| 107 |
+
fake = _stub_sarvam(side_effect=asyncio.CancelledError())
|
| 108 |
+
with self.assertRaises(asyncio.CancelledError):
|
| 109 |
+
asyncio.run(translate_to_english(HINDI_QUERY, sarvam=fake))
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
class TranslateToIndicTests(unittest.TestCase):
|
| 113 |
+
def test_happy_path(self):
|
| 114 |
+
fake = _stub_sarvam(return_value=LLMResult(
|
| 115 |
+
text="Haan, HDFC ERGO Optima Secure mein Ayurveda cover hai.",
|
| 116 |
+
model="sarvam-m",
|
| 117 |
+
))
|
| 118 |
+
out = asyncio.run(translate_to_indic(ENGLISH_REPLY, sarvam=fake))
|
| 119 |
+
self.assertIn("HDFC", out)
|
| 120 |
+
|
| 121 |
+
def test_passthrough_on_httpx_read_timeout(self):
|
| 122 |
+
fake = _stub_sarvam(side_effect=httpx.ReadTimeout(
|
| 123 |
+
"Read timeout", request=_make_request(),
|
| 124 |
+
))
|
| 125 |
+
out = asyncio.run(translate_to_indic(ENGLISH_REPLY, sarvam=fake))
|
| 126 |
+
self.assertEqual(out, ENGLISH_REPLY)
|
| 127 |
+
|
| 128 |
+
def test_passthrough_on_http_status_error(self):
|
| 129 |
+
req = _make_request()
|
| 130 |
+
resp = httpx.Response(status_code=500, request=req)
|
| 131 |
+
fake = _stub_sarvam(side_effect=httpx.HTTPStatusError(
|
| 132 |
+
"Server Error", request=req, response=resp,
|
| 133 |
+
))
|
| 134 |
+
out = asyncio.run(translate_to_indic(ENGLISH_REPLY, sarvam=fake))
|
| 135 |
+
self.assertEqual(out, ENGLISH_REPLY)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
if __name__ == "__main__":
|
| 139 |
+
unittest.main()
|