Sync Dream QA latency polish

dream_customs/app_logic.py

@@ -35,15 +35,15 @@ from dream_customs.schema import CustomsSession
 
 DEFAULT_TEXT_MODEL = "hf.co/openbmb/MiniCPM5-1B-GGUF:Q8_0"
 DEFAULT_VISION_MODEL = "openbmb/minicpm-v4.6"
-DEFAULT_HOSTED_TIMEOUT_SECONDS = 60.0
-DEFAULT_ASR_TIMEOUT_SECONDS = 45.0
+DEFAULT_HOSTED_TIMEOUT_SECONDS = 9.0
+DEFAULT_ASR_TIMEOUT_SECONDS = 9.0
 DEFAULT_TEXT_TEMPERATURE = 0.0
 DEFAULT_VISION_TEMPERATURE = 0.1
-DEFAULT_TEXT_MAX_TOKENS = 560
-DEFAULT_VISION_MAX_TOKENS = 320
-DEFAULT_TEXT_LATENCY_BUDGET_MS = 3500
-DEFAULT_VISION_LATENCY_BUDGET_MS = 6500
-DEFAULT_ASR_LATENCY_BUDGET_MS = 2500
+DEFAULT_TEXT_MAX_TOKENS = 480
+DEFAULT_VISION_MAX_TOKENS = 220
+DEFAULT_TEXT_LATENCY_BUDGET_MS = 8000
+DEFAULT_VISION_LATENCY_BUDGET_MS = 9000
+DEFAULT_ASR_LATENCY_BUDGET_MS = 8000
 
 
 def _file_path(value: Any) -> str:
@@ -154,6 +154,7 @@ def _clients(text_backend: str, vision_backend: str, **settings):
             timeout=resolved["text_timeout_seconds"],
             temperature=resolved["text_temperature"],
             max_tokens=resolved["text_max_tokens"],
+            latency_budget_ms=resolved["text_latency_budget_ms"],
         )
     else:
         text_client = FakeTextClient()
@@ -171,6 +172,7 @@ def _clients(text_backend: str, vision_backend: str, **settings):
             timeout=resolved["vision_timeout_seconds"],
             temperature=resolved["vision_temperature"],
             max_tokens=resolved["vision_max_tokens"],
+            latency_budget_ms=resolved["vision_latency_budget_ms"],
         )
     else:
         vision_client = FakeVisionClient()
@@ -180,6 +182,7 @@ def _clients(text_backend: str, vision_backend: str, **settings):
             endpoint=resolved["asr_endpoint"],
             token=resolved["hosted_token"],
             timeout=resolved["asr_timeout_seconds"],
+            latency_budget_ms=resolved["asr_latency_budget_ms"],
         )
     else:
         asr_client = FakeASRClient()

modal_backend/contracts.py

@@ -24,12 +24,12 @@ def normalize_text_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
                     if content:
                         parts.append(content)
             prompt = "\n".join(parts).strip()
-    max_tokens = payload.get("max_tokens", 560)
+    max_tokens = payload.get("max_tokens", 480)
     try:
         max_tokens = int(max_tokens)
     except (TypeError, ValueError):
-        max_tokens = 560
-    max_tokens = max(64, min(max_tokens, 1200))
+        max_tokens = 480
+    max_tokens = max(64, min(max_tokens, 700))
     temperature = payload.get("temperature", 0.0)
     try:
         temperature = float(temperature)

modal_backend/dream_customs_modal.py

@@ -538,7 +538,8 @@ async def vision(
                 ],
             }
         ]
-        result = pipe(text=messages, max_new_tokens=int(payload.get("max_tokens", 320)))
+        max_tokens = max(64, min(int(payload.get("max_tokens", 220)), 400))
+        result = pipe(text=messages, max_new_tokens=max_tokens)
     return response_payload(_stringify_pipeline_result(result))
 
 

scripts/benchmark_hosted_latency.py

@@ -0,0 +1,188 @@
+import json
+import os
+import sys
+import tempfile
+import time
+import wave
+from pathlib import Path
+from typing import Any, Callable
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from dream_customs.models import HostedASRClient, HostedMiniCPMTextClient, HostedMiniCPMVisionClient
+from dream_customs.prompts import negotiation_prompt, today_tip_prompt
+from dream_customs.pipeline import build_intake, build_qa_state
+from dream_customs.schema import TodayTipCard
+
+
+class TextFallback:
+    def generate_negotiation(self, _prompt: str) -> dict[str, Any]:
+        return {"visitor_name": "fallback", "questions": ["fallback"], "tone_note": "fallback"}
+
+    def generate_today_tip(self, _prompt: str) -> TodayTipCard:
+        return TodayTipCard(
+            dream_summary="fallback",
+            main_question="fallback",
+            dream_anchors=["fallback"],
+            followup_questions=[],
+            user_answers=[],
+            interpretation="fallback",
+            today_tip="fallback",
+            tiny_action="fallback",
+            caring_note="fallback",
+            safety_note="",
+        )
+
+
+class VisionFallback:
+    def extract_clues(self, _image_path: str) -> list[str]:
+        return ["fallback"]
+
+    def extract_witness(self, _image_path: str):
+        raise RuntimeError("fallback witness should not be used in this benchmark")
+
+
+class ASRFallback:
+    def transcribe(self, _audio_path: str) -> str:
+        return "fallback"
+
+
+def _measure(name: str, fn: Callable[[], Any]) -> dict[str, Any]:
+    start = time.perf_counter()
+    try:
+        value = fn()
+        ok = True
+        error = ""
+    except Exception as exc:  # pragma: no cover - benchmark safety net
+        value = None
+        ok = False
+        error = exc.__class__.__name__
+    elapsed = time.perf_counter() - start
+    return {
+        "name": name,
+        "ok": ok,
+        "elapsed_seconds": round(elapsed, 3),
+        "fallback": _looks_like_fallback(value),
+        "error": error,
+    }
+
+
+def _looks_like_fallback(value: Any) -> bool:
+    if isinstance(value, dict):
+        return value.get("visitor_name") == "fallback"
+    if isinstance(value, TodayTipCard):
+        return value.dream_summary == "fallback"
+    if isinstance(value, list):
+        return value == ["fallback"]
+    return value == "fallback"
+
+
+def _write_probe_wav() -> str:
+    temp = tempfile.NamedTemporaryFile(delete=False, suffix=".wav")
+    temp.close()
+    with wave.open(temp.name, "wb") as wav:
+        wav.setnchannels(1)
+        wav.setsampwidth(2)
+        wav.setframerate(16000)
+        wav.writeframes(b"\x00\x00" * 1600)
+    return temp.name
+
+
+def main() -> int:
+    token = os.getenv("DREAM_CUSTOMS_HOSTED_TOKEN", "")
+    text_endpoint = os.getenv("DREAM_CUSTOMS_TEXT_ENDPOINT", "").strip()
+    vision_endpoint = os.getenv("DREAM_CUSTOMS_VISION_ENDPOINT", "").strip()
+    asr_endpoint = os.getenv("DREAM_CUSTOMS_ASR_ENDPOINT", "").strip()
+    image_path = os.getenv("DREAM_CUSTOMS_SMOKE_IMAGE", "").strip()
+    audio_path = os.getenv("DREAM_CUSTOMS_SMOKE_AUDIO", "").strip()
+
+    text_timeout = float(os.getenv("DREAM_CUSTOMS_BENCH_TEXT_TIMEOUT", "9"))
+    vision_timeout = float(os.getenv("DREAM_CUSTOMS_BENCH_VISION_TIMEOUT", "9"))
+    asr_timeout = float(os.getenv("DREAM_CUSTOMS_BENCH_ASR_TIMEOUT", "9"))
+    text_budget = int(float(os.getenv("DREAM_CUSTOMS_BENCH_TEXT_BUDGET_MS", "8000")))
+    vision_budget = int(float(os.getenv("DREAM_CUSTOMS_BENCH_VISION_BUDGET_MS", "9000")))
+    asr_budget = int(float(os.getenv("DREAM_CUSTOMS_BENCH_ASR_BUDGET_MS", "8000")))
+
+    intake = build_intake(
+        dream_text="I dreamed my phone died while I waited for an elevator.",
+        mood="Anxious",
+    )
+    answers = "The dead phone felt closest to being behind before I even start."
+    state = build_qa_state(
+        intake,
+        questions=["Which detail feels closest to your waking life right now?"],
+        answers=[answers],
+        language="en",
+    )
+
+    report: dict[str, Any] = {
+        "configured": {
+            "text_endpoint": bool(text_endpoint),
+            "vision_endpoint": bool(vision_endpoint),
+            "asr_endpoint": bool(asr_endpoint),
+            "token": bool(token),
+            "image_path": bool(image_path),
+            "audio_path": bool(audio_path),
+        },
+        "budgets_ms": {"text": text_budget, "vision": vision_budget, "asr": asr_budget},
+        "results": [],
+    }
+
+    if text_endpoint:
+        text_client = HostedMiniCPMTextClient(
+            endpoint=text_endpoint,
+            token=token,
+            timeout=text_timeout,
+            max_tokens=560,
+            latency_budget_ms=text_budget,
+            fallback=TextFallback(),
+        )
+        report["results"].append(
+            _measure(
+                "text_negotiation",
+                lambda: text_client.generate_negotiation(negotiation_prompt(intake, "en")),
+            )
+        )
+        report["results"].append(
+            _measure(
+                "text_today_tip",
+                lambda: text_client.generate_today_tip(today_tip_prompt(state, "en")),
+            )
+        )
+    if vision_endpoint and image_path and Path(image_path).exists():
+        vision_client = HostedMiniCPMVisionClient(
+            endpoint=vision_endpoint,
+            token=token,
+            timeout=vision_timeout,
+            max_tokens=220,
+            latency_budget_ms=vision_budget,
+            fallback=VisionFallback(),
+        )
+        report["results"].append(_measure("vision_clues", lambda: vision_client.extract_clues(image_path)))
+    if asr_endpoint:
+        probe_audio = ""
+        try:
+            probe_audio = audio_path if audio_path and Path(audio_path).exists() else _write_probe_wav()
+            asr_client = HostedASRClient(
+                endpoint=asr_endpoint,
+                token=token,
+                timeout=asr_timeout,
+                latency_budget_ms=asr_budget,
+                fallback=ASRFallback(),
+            )
+            report["results"].append(_measure("asr_transcribe", lambda: asr_client.transcribe(probe_audio)))
+        finally:
+            if probe_audio and probe_audio != audio_path:
+                try:
+                    os.unlink(probe_audio)
+                except OSError:
+                    pass
+
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_app_logic.py

@@ -125,6 +125,9 @@ def test_developer_settings_configure_hosted_clients():
         vision_temperature=0.22,
         text_max_tokens=256,
         vision_max_tokens=128,
+        text_latency_budget_ms=5000,
+        vision_latency_budget_ms=6000,
+        asr_latency_budget_ms=4000,
     )
 
     assert isinstance(text_client, HostedMiniCPMTextClient)
@@ -140,6 +143,25 @@ def test_developer_settings_configure_hosted_clients():
     assert vision_client.temperature == 0.22
     assert text_client.max_tokens == 256
     assert vision_client.max_tokens == 128
+    assert text_client.latency_budget_ms == 5000
+    assert vision_client.latency_budget_ms == 6000
+    assert asr_client.latency_budget_ms == 4000
+    assert text_client.request_timeout == 5.0
+    assert vision_client.request_timeout == 6.0
+    assert asr_client.request_timeout == 4.0
+
+
+def test_default_modal_latency_settings_target_fast_feedback():
+    settings = _client_settings()
+
+    assert settings["text_timeout_seconds"] <= 10
+    assert settings["vision_timeout_seconds"] <= 10
+    assert settings["asr_timeout_seconds"] <= 10
+    assert settings["text_latency_budget_ms"] <= 10_000
+    assert settings["vision_latency_budget_ms"] <= 10_000
+    assert settings["asr_latency_budget_ms"] <= 10_000
+    assert settings["text_max_tokens"] <= 560
+    assert settings["vision_max_tokens"] <= 220
 
 
 def test_asr_endpoint_derives_from_modal_text_endpoint_when_secret_is_missing():

tests/test_modal_contract.py

@@ -22,12 +22,12 @@ def test_normalize_text_payload_accepts_openai_style_messages():
         {"messages": [{"role": "user", "content": "Dream of an elevator."}]}
     )
     assert payload["prompt"] == "Dream of an elevator."
-    assert payload["max_tokens"] == 560
+    assert payload["max_tokens"] == 480
 
 
 def test_normalize_text_payload_clamps_max_tokens():
     payload = normalize_text_payload({"prompt": "x", "max_tokens": 9000})
-    assert payload["max_tokens"] == 1200
+    assert payload["max_tokens"] == 700
 
 
 def test_decode_image_payload_accepts_image_key():

tests/test_ui_actions.py

@@ -332,7 +332,7 @@ def test_mobile_reset_restores_calm_mood():
         60,
         0.2,
         0.1,
-        560,
+        480,
         320,
         "demo",
         "",