mechramc commited on
Commit
39f178d
Β·
verified Β·
1 Parent(s): 09d72e9

v1.2.0: optional ElevenLabs cloud voice (onboarding key box, local Piper fallback)

Browse files
chittios/main.py CHANGED
@@ -1,706 +1,790 @@
1
- """ChittiOS β€” the Reachy Mini app shell (Phase 7: the composed OS).
2
-
3
- Phase 1 proved the publish β†’ install β†’ run pipeline with a greeting and idle
4
- breathing. This is the real app: a thin shell that constructs the *concrete*,
5
- hardware- and model-backed engines and hands them to ``chittios_core``'s
6
- :class:`~chittios_core.orchestrator.ChittiOS`, which owns every line of composition
7
- logic. Nothing here decides *what* ChittiOS does β€” it only supplies the real body,
8
- the real camera and microphone, and the real STT/LLM/TTS to the seams the
9
- orchestrator drives, exactly where a test supplies stubs.
10
-
11
- Thin, and gracefully degrading
12
- ------------------------------
13
- Every heavy engine is optional: a fresh SBC may lack a Piper voice, a Whisper
14
- model, or a reachable LLM, and the household should still get a robot that
15
- recognises them and says hello rather than one that refuses to boot. So each
16
- engine is built behind a small guarded factory that returns ``None`` (or a stub
17
- fallback for the one engine β€” TTS β€” the greeting cannot do without) when its
18
- dependency or device is absent. A missing LLM costs conversation and nothing
19
- else; perception, greeting, and idle motion run regardless.
20
-
21
- Idle breathing stays a raw-SDK control loop
22
- --------------------------------------------
23
- The "alive" breathing is the one thing that cannot go through the HAL motion
24
- primitives: it needs ``set_target`` at a fixed 60 Hz (repeated ``goto_target``
25
- each restart the interpolation β€” the judder Phase 1 diagnosed). So it stays here,
26
- against the raw SDK, wrapped as an :class:`IdleMotion` the orchestrator runs on
27
- its own thread β€” the reference control loop, unchanged, just handed to the
28
- composition instead of being ``run``'s only body.
29
- """
30
-
31
- from __future__ import annotations
32
-
33
- import logging
34
- import os
35
- import threading
36
- import time
37
- from collections.abc import Callable
38
- from contextlib import ExitStack
39
- from datetime import UTC, datetime
40
- from pathlib import Path
41
- from typing import TypeVar
42
-
43
- import numpy as np
44
- from fastapi import FastAPI
45
- from numpy.typing import NDArray
46
- from pydantic import BaseModel
47
- from reachy_mini import ReachyMini, ReachyMiniApp
48
- from reachy_mini.utils import create_head_pose
49
-
50
- from chittios_core.context import ContextManager
51
- from chittios_core.conversation.config import LlmConfig, load_llm_config, write_llm_config
52
- from chittios_core.conversation.llm import OpenAICompatibleLlm
53
- from chittios_core.conversation.stt import WhisperStt
54
- from chittios_core.conversation.vad import OnnxVadSegmenter
55
- from chittios_core.expression.tts import PiperTtsEngine, StubTtsEngine, TtsEngine
56
- from chittios_core.identity import roster
57
- from chittios_core.identity.face.detector import FaceDetectorProtocol, YuNetDetector
58
- from chittios_core.identity.face.embedder import SFaceEmbedder
59
- from chittios_core.identity.face.provider import FaceIdentityProvider
60
- from chittios_core.identity.onboarding.adapters import (
61
- AntennaHoldSignal,
62
- MicNameRecorder,
63
- ModelEnrollerFactory,
64
- PassphraseCustody,
65
- PullFaceSource,
66
- PullVoiceSource,
67
- SyncSpeechOut,
68
- WhisperSpeechIn,
69
- )
70
- from chittios_core.identity.onboarding.authority import AuthorityGate
71
- from chittios_core.identity.onboarding.conductor import CaptureConductor
72
- from chittios_core.identity.onboarding.dialog import (
73
- DialogOutcome,
74
- DialogResult,
75
- EnrollmentDialog,
76
- )
77
- from chittios_core.identity.onboarding.ports import SessionRecognizer, SpeechOut
78
- from chittios_core.identity.session import Session
79
- from chittios_core.identity.store.enrollments import EnrollmentStore
80
- from chittios_core.identity.store.keys import (
81
- FileKeyStore,
82
- KeyCustodyError,
83
- KeyringKeyStore,
84
- get_or_create_key,
85
- )
86
- from chittios_core.identity.types import IdentityClaim, IdentityProvider, Modality
87
- from chittios_core.identity.voice.embedder import ECAPAEmbedder
88
- from chittios_core.identity.voice.provider import VoiceIdentityProvider
89
- from chittios_core.identity.voice.segmenter import SegmenterProtocol, SpeechSegment
90
- from chittios_core.orchestrator import (
91
- ChittiOS,
92
- ConversationSeams,
93
- OwnerEnrollment,
94
- PerceptionSeams,
95
- )
96
- from chittios_core.perception.loop import FacePipeline
97
- from chittios_core.perception.sources import (
98
- CameraSource,
99
- MicSource,
100
- ReachyMiniCameraSource,
101
- SoundDeviceMicSource,
102
- )
103
- from hal.reachy_mini.adapter import ReachyMiniAdapter
104
-
105
- logger = logging.getLogger(__name__)
106
-
107
- _T = TypeVar("_T")
108
-
109
- #: Where the enrolled household lives on the robot. The store creates its parent.
110
- STORE_PATH = Path.home() / ".local" / "share" / "chittios" / "enrollments.db"
111
-
112
- #: Where the recoverable passphrase custody vault lives β€” the Argon2id-wrapped
113
- #: data key the recovery flow re-derives. Beside the store because a wrapped key
114
- #: is ciphertext, useless without the household passphrase (see vault.py).
115
- VAULT_PATH = Path.home() / ".local" / "share" / "chittios" / "passphrase.vault"
116
-
117
- #: Piper voice model. Prefer the one bundled with the app so a fresh robot speaks
118
- #: real words out of the box; fall back to a user-supplied model in the data dir.
119
- _BUNDLED_VOICE = Path(__file__).parent / "assets" / "voice.onnx"
120
- _USER_VOICE = Path.home() / ".local" / "share" / "chittios" / "voice.onnx"
121
-
122
- #: Spoken by the empty-store fallback when the robot cannot run onboarding β€”
123
- #: honest and warm rather than silent. The robot then idles (breathing) rather
124
- #: than exiting, so a household that unboxes it into a half-provisioned state
125
- #: still meets something alive.
126
- ONBOARDING_UNAVAILABLE_GREETING = (
127
- "Hello. I'm not quite ready to set up your household yet, but I'm here with you."
128
- )
129
-
130
- # ── idle breathing (the reference 60 Hz control loop) ──────────────────────
131
-
132
- #: Control-loop rate. The fleet app uses 60 Hz; the docs floor smoothness at
133
- #: 30 Hz. One ``set_target`` per tick, period held by adaptive sleep.
134
- CONTROL_FREQUENCY_HZ = 60.0
135
- CONTROL_PERIOD_S = 1.0 / CONTROL_FREQUENCY_HZ
136
-
137
- #: Idle "breathing": the target pose is a smooth sine of time (not discrete
138
- #: commands). Amplitudes kept small and slow, matching the reference breathing.
139
- BREATHING_PITCH_DEG = 3.0
140
- BREATHING_PITCH_HZ = 0.2
141
- LOOK_YAW_DEG = 8.0
142
- LOOK_YAW_HZ = 0.05
143
-
144
-
145
- class BreathingMotion:
146
- """The idle "alive" motion, as an :class:`IdleMotion` the orchestrator runs.
147
-
148
- The Phase 1 control loop, unchanged in substance: one ``set_target`` per tick
149
- at a monotonic-aligned 60 Hz, the pose a smooth sine of time. It lives against
150
- the raw SDK rather than the HAL because it needs ``set_target``'s immediacy β€”
151
- the HAL's queued ``goto_target`` would restart interpolation every tick and
152
- judder. The orchestrator runs this on its own thread and stops it with the same
153
- event that stops everything else.
154
-
155
- Args:
156
- mini: The connected robot to drive.
157
- """
158
-
159
- def __init__(self, mini: ReachyMini) -> None:
160
- self._mini = mini
161
-
162
- def run(self, stop: threading.Event) -> None:
163
- """Breathe until ``stop`` is set, then return."""
164
- start = time.monotonic()
165
- while not stop.is_set():
166
- tick_start = time.monotonic()
167
- elapsed = tick_start - start
168
-
169
- pitch_deg = BREATHING_PITCH_DEG * np.sin(2.0 * np.pi * BREATHING_PITCH_HZ * elapsed)
170
- yaw_deg = LOOK_YAW_DEG * np.sin(2.0 * np.pi * LOOK_YAW_HZ * elapsed)
171
- self._mini.set_target(head=create_head_pose(pitch=pitch_deg, yaw=yaw_deg, degrees=True))
172
-
173
- # Adaptive sleep holds the period regardless of per-tick cost; the
174
- # event doubles as an interruptible sleep, so a stop returns at once.
175
- sleep_s = max(0.0, CONTROL_PERIOD_S - (time.monotonic() - tick_start))
176
- if stop.wait(sleep_s):
177
- break
178
-
179
-
180
- class _GreetAndIdleFallback:
181
- """The empty-store path when onboarding genuinely cannot be assembled.
182
-
183
- An unboxed robot has an empty store and, normally, runs the spoken owner
184
- enrollment. But if the pieces that ritual needs β€” a working microphone, the
185
- voice models, the vault β€” cannot be built, the honest failure is not to exit
186
- silently (a robot that does nothing looks broken) but to *say so* and stay
187
- alive. Staying alive is the orchestrator's job now: it breathes on its own
188
- thread and offers ``enroll_owner`` again on a cadence for the whole empty-store
189
- lifetime, so this only owes a single greeting. It says its line the first time
190
- it is offered and returns ``FAILED``; subsequent offers return ``FAILED``
191
- silently, so the robot breathes on quietly rather than repeating itself.
192
-
193
- It satisfies :class:`OwnerEnrollment` structurally, so the orchestrator drives
194
- it through exactly the same ``enroll_owner`` seam it drives the real dialog
195
- through β€” the orchestrator's own empty-store semantics are untouched; only
196
- *what* is wired into that seam differs when the dialog could not be built.
197
-
198
- Args:
199
- speech_out: How the greeting is spoken (the same sync TTS seam the dialog
200
- would have used).
201
- greeting: The single line spoken on the first offer.
202
- """
203
-
204
- def __init__(
205
- self,
206
- speech_out: SpeechOut,
207
- *,
208
- greeting: str = ONBOARDING_UNAVAILABLE_GREETING,
209
- ) -> None:
210
- self._speech_out = speech_out
211
- self._greeting = greeting
212
- self._greeted = False
213
-
214
- def enroll_owner(self) -> DialogResult:
215
- """Greet on the first offer, then stay quiet. Nothing is ever enrolled."""
216
- if not self._greeted:
217
- self._greeted = True
218
- try:
219
- self._speech_out.speak(self._greeting)
220
- except Exception:
221
- logger.warning("the fallback greeting could not be spoken", exc_info=True)
222
- return DialogResult(DialogOutcome.FAILED)
223
-
224
-
225
- # ── null sources (degrade when a device is absent) ─────────────────────────
226
-
227
-
228
- class _NullCamera:
229
- """A :class:`CameraSource` that always sees nothing.
230
-
231
- Used when no camera can be opened: perception then runs on voice alone rather
232
- than failing to start. A returned ``None`` is the source contract's ordinary
233
- "nothing right now", so the loop simply never gets a face.
234
- """
235
-
236
- def read(self) -> None:
237
- return None
238
-
239
-
240
- class _NullMic:
241
- """A :class:`MicSource` that always hears nothing β€” the audio counterpart."""
242
-
243
- def read(self) -> None:
244
- return None
245
-
246
-
247
- class _NullVoiceProvider:
248
- """A voice :class:`IdentityProvider` that recognises no one β€” face-only builds.
249
-
250
- The speaker encoder (ECAPA) is torch- and speechbrain-backed, ~2 GB that does
251
- not fit on the Reachy Mini, so voice **identification** is dropped on the
252
- robot (the owner's call). Perception still segments audio β€” the conversation
253
- loop needs those utterances for STT β€” but the voice *provider* it drives is
254
- this: every utterance returns an unknown claim, exactly as the real provider
255
- does for anyone unenrolled, so the fusion and authorization paths downstream
256
- are unchanged and simply never receive a voice match. Recognition runs on the
257
- face alone.
258
-
259
- It satisfies :class:`~chittios_core.identity.types.IdentityProvider`\\
260
- ``[SpeechSegment]`` structurally, so it drops into the same perception seam
261
- the real provider fills, and it constructs nothing torch-backed.
262
- """
263
-
264
- @property
265
- def modality(self) -> Modality:
266
- return Modality.VOICE
267
-
268
- def identify(
269
- self, observation: SpeechSegment, /, *, observed_at: datetime | None = None
270
- ) -> IdentityClaim:
271
- """Return an unknown voice claim for any utterance β€” no one is identified."""
272
- return IdentityClaim.unknown(
273
- modality=Modality.VOICE, observed_at=observed_at or datetime.now(UTC)
274
- )
275
-
276
-
277
- class _NullVoiceEmbedder:
278
- """A voice embedder that is never called β€” the placeholder for face-only sittings.
279
-
280
- ``ModelEnrollerFactory`` holds a voice embedder so it can mint voice enrollers,
281
- but a face-only enrollment (``enroll_voice=False``) never asks it for one, so
282
- this stand-in is only ever *held*, never *invoked*. It exists so the factory's
283
- wiring is uniform whether or not a speaker encoder is present, without
284
- constructing the torch-backed ECAPA on a robot that cannot run it. Being
285
- called would mean a face-only sitting tried to embed voice after all, which is
286
- a wiring bug worth failing loudly on rather than papering over with a vector.
287
- """
288
-
289
- def embed(self, canonical_audio: NDArray[np.float32], /) -> NDArray[np.float32]:
290
- raise NotImplementedError(
291
- "voice enrollment is disabled on this build; _NullVoiceEmbedder must never be called"
292
- )
293
-
294
-
295
- def _guard(what: str, build: Callable[[], _T]) -> _T | None:
296
- """Build an optional engine, degrading to ``None`` if its runtime is absent.
297
-
298
- The one place the "a missing model must not stop the robot booting" rule is
299
- enforced: any failure to construct β€” a missing package, an absent model file,
300
- an unopenable device β€” is logged and turned into ``None`` rather than raised.
301
- """
302
- try:
303
- return build()
304
- except Exception:
305
- logger.warning("%s is unavailable; ChittiOS will run without it", what, exc_info=True)
306
- return None
307
-
308
-
309
- def _resolve_store_key() -> bytes:
310
- """The enrollment store's encryption key: OS keychain where one exists, else a
311
- mode-0600 key file on a headless Linux host.
312
-
313
- The robot is headless Linux with no login keyring, so ``KeyringKeyStore``
314
- raises there. ``FileKeyStore`` keeps the key in a 0600 file beside the store β€”
315
- weaker than a keychain (the key sits near the data it protects), but on a box
316
- with no keychain it is that or no at-rest encryption at all. Windows and macOS
317
- keep the keychain; ``FileKeyStore`` is POSIX-only, so this only falls back on
318
- a POSIX host.
319
- """
320
- try:
321
- return get_or_create_key(KeyringKeyStore())
322
- except KeyCustodyError:
323
- if os.name != "posix":
324
- raise
325
- key_dir = STORE_PATH.parent
326
- key_dir.mkdir(parents=True, exist_ok=True)
327
- logger.warning("no OS keychain available; using a 0600 key file in %s", key_dir)
328
- return get_or_create_key(FileKeyStore(key_dir))
329
-
330
-
331
- class LlmSettingsUpdate(BaseModel):
332
- """The dashboard's ``POST /llm-config`` body β€” the three settings, plus key semantics.
333
-
334
- ``api_key`` distinguishes three intents a single password field cannot
335
- otherwise express: ``None`` (field left blank) leaves the stored key
336
- untouched, an empty string clears it, and any other string sets it β€” so
337
- changing the URL never silently drops a previously saved key.
338
-
339
- Defined at module scope, not inside :meth:`Chittios._register_settings_routes`,
340
- on purpose: this module uses ``from __future__ import annotations`` (PEP 563),
341
- so the route handler's ``update: LlmSettingsUpdate`` annotation is a *string*
342
- that FastAPI resolves with :func:`typing.get_type_hints` against the module's
343
- globals. A method-local class is not in those globals, so the annotation
344
- fails to resolve to this Pydantic model and FastAPI falls back to treating
345
- ``update`` as a required *query* parameter β€” which no JSON body can satisfy,
346
- yielding ``422 Unprocessable Entity`` on every save. Keeping the model here,
347
- where ``get_type_hints`` can find it, is what makes the body bind.
348
- """
349
-
350
- base_url: str
351
- model: str
352
- api_key: str | None = None
353
-
354
-
355
- class Chittios(ReachyMiniApp): # type: ignore[misc] # reachy_mini ships no stubs; base resolves to Any
356
- # A small settings web UI, served by the daemon on this URL, lets a household
357
- # point ChittiOS at their own LLM endpoint from the robot dashboard. The base
358
- # class builds ``self.settings_app`` (a FastAPI app) from this URL; we add the
359
- # config routes in ``run``. ``0.0.0.0`` so the dashboard reaches it over the LAN.
360
- custom_app_url: str | None = "http://0.0.0.0:8042"
361
- request_media_backend: str | None = None
362
-
363
- def __init__(self, running_on_wireless: bool = False) -> None:
364
- """Build the app and register the settings routes before the server serves.
365
-
366
- The base class builds ``self.settings_app`` here and ``wrapped_run`` starts
367
- its web-server thread *before* it calls :meth:`run`. Registering the config
368
- routes inside ``run`` (as this once did) therefore leaves a window β€” the
369
- whole time the robot spends connecting to the daemon before ``run`` begins β€”
370
- during which the server is already accepting requests but ``/llm-config``
371
- does not exist yet, so the dashboard's on-load fetch races ahead of the
372
- route and gets a 404. Registering them now, at construction, closes that
373
- window: the routes are on the app before the first request can arrive.
374
- """
375
- super().__init__(running_on_wireless)
376
- self._register_settings_routes()
377
-
378
- def run(self, reachy_mini: ReachyMini, stop_event: threading.Event) -> None:
379
- """Construct the real engines and hand them to the orchestrator.
380
-
381
- Opens the camera and microphone for the app's lifetime (released on
382
- return), builds whatever recognition, conversation, and speech engines the
383
- machine can provide, and lets :class:`ChittiOS` compose and run them until
384
- the daemon sets ``stop_event``.
385
- """
386
- logging.basicConfig(level=logging.INFO)
387
- body = ReachyMiniAdapter(reachy_mini)
388
- store = EnrollmentStore(STORE_PATH, _resolve_store_key())
389
- tts = self._build_tts()
390
-
391
- with ExitStack() as sources:
392
- # The camera and microphone are shared: on an empty store only
393
- # onboarding reads them, on a populated one only perception does, and
394
- # the two paths never run together β€” so one source each, not two
395
- # fighting over an exclusive device.
396
- #
397
- # The camera comes from the SDK, not OpenCV: on the robot the daemon
398
- # owns the Orbbec and streams frames over a GStreamer IPC endpoint, so
399
- # a direct ``cv2.VideoCapture(0)`` fails ("could not open camera 0").
400
- # ``reachy_mini.media.get_frame()`` is how an app gets frames while the
401
- # daemon holds the device, and it already returns exactly the BGR
402
- # frames the perception loop expects.
403
- camera: CameraSource = (
404
- _guard("camera", lambda: ReachyMiniCameraSource(reachy_mini.media)) or _NullCamera()
405
- )
406
- mic: MicSource = (
407
- _guard("microphone", lambda: sources.enter_context(SoundDeviceMicSource()))
408
- or _NullMic()
409
- )
410
- # The heavy recognition models, built once and shared by perception and
411
- # onboarding so neither boot path pays to load them twice.
412
- detector = YuNetDetector()
413
- segmenter = OnnxVadSegmenter()
414
- face_embedder = SFaceEmbedder()
415
- # Voice identification is the one torch-backed engine left on this
416
- # path: ECAPA pulls speechbrain and torch (~2 GB), which do not fit the
417
- # Reachy Mini. Behind _guard it degrades to None when they are absent β€”
418
- # recognition then runs on the face alone (_NullVoiceProvider) and the
419
- # first-boot sitting enrols face-only. On a dev box with the `voice`
420
- # extra installed it builds as before and voice-ID is fully live.
421
- voice_embedder = _guard("voice identification (ECAPA)", ECAPAEmbedder)
422
- # ``tiny.en`` rather than ``base``: on this SBC's CPU, ``base`` spends
423
- # ~12 s decoding a 2 s turn, so utterances queue faster than they clear
424
- # and a reply lands a minute late β€” unusable. The English-only ``tiny``
425
- # is several times faster for a modest accuracy cost that short
426
- # household turns tolerate well, keeping time-to-answer conversational.
427
- #
428
- # The ``.en`` build also *is* the language lock: forcing English keeps
429
- # faster-whisper from auto-detecting per utterance, which on short,
430
- # quiet turns β€” a name, a yes β€” has mis-fired to Italian on the robot
431
- # and garbled every transcript. ``language="en"`` is kept as belt and
432
- # braces. The core WhisperStt default stays language-agnostic; only the
433
- # app pins the model and the language ChittiOS speaks.
434
- stt = _guard("Whisper STT", lambda: WhisperStt("tiny.en", language="en"))
435
-
436
- perception = self._build_perception(
437
- store, camera, mic, detector, segmenter, face_embedder, voice_embedder
438
- )
439
- conversation = ConversationSeams(stt=stt, llm=_guard("LLM", self._build_llm))
440
- onboarding = self._build_onboarding(
441
- store=store,
442
- body=body,
443
- tts=tts,
444
- reachy_mini=reachy_mini,
445
- camera=camera,
446
- mic=mic,
447
- detector=detector,
448
- segmenter=segmenter,
449
- face_embedder=face_embedder,
450
- voice_embedder=voice_embedder,
451
- stt=stt,
452
- stop_event=stop_event,
453
- )
454
-
455
- orchestrator = ChittiOS(
456
- store=store,
457
- perception=perception,
458
- conversation=conversation,
459
- tts=tts,
460
- context=ContextManager(),
461
- idle_motion=BreathingMotion(reachy_mini),
462
- onboarding=onboarding,
463
- )
464
- orchestrator.run(body, stop_event=stop_event)
465
-
466
- body.close()
467
-
468
- def _build_llm(self) -> OpenAICompatibleLlm:
469
- """Construct the conversation LLM from the household's resolved config.
470
-
471
- The endpoint is not hardcoded: :func:`load_llm_config` resolves it from
472
- the environment, then the config file the settings UI writes, then the
473
- built-in default (the Spark's Gemma). Whatever comes back is where this
474
- robot sends conversation transcripts β€” local by default, or wherever the
475
- household pointed it. Built behind ``_guard`` in :meth:`run`, so a
476
- malformed config or missing HTTP stack costs conversation and nothing else.
477
- """
478
- config = load_llm_config()
479
- logger.info("LLM endpoint: %s (model %s)", config.base_url, config.model)
480
- return OpenAICompatibleLlm(
481
- base_url=config.base_url,
482
- model=config.model,
483
- api_key=config.api_key,
484
- )
485
-
486
- def _register_settings_routes(self) -> None:
487
- """Add the LLM-endpoint settings routes to the dashboard's settings app.
488
-
489
- The base class builds ``self.settings_app`` from ``custom_app_url`` and
490
- serves ``static/index.html`` as the dashboard page; here we give that page
491
- an API. ``GET /llm-config`` reports the resolved endpoint and whether a key
492
- is set (never the key itself β€” a secret is not echoed back over HTTP);
493
- ``POST /llm-config`` writes the household's choice to the config file.
494
-
495
- Writes take effect on the next app start β€” there is no live client swap, by
496
- design: rebuilding the conversation engine mid-turn is far more surface
497
- area than the feature needs, so the contract is simply "restart to apply".
498
- """
499
- # The base class exposes ``settings_app`` untyped (reachy_mini ships no
500
- # stubs); bind it to a concrete ``FastAPI`` local so its route decorators
501
- # are typed rather than ``Any`` (which mypy rejects as an untyped decorator).
502
- settings_app: FastAPI | None = self.settings_app
503
- if settings_app is None: # webserver disabled; nothing to wire.
504
- return
505
-
506
- @settings_app.get("/llm-config")
507
- def get_llm_config() -> dict[str, object]:
508
- """Report the resolved endpoint for the dashboard to display."""
509
- config = load_llm_config()
510
- return {
511
- "base_url": config.base_url,
512
- "model": config.model,
513
- "has_api_key": config.api_key is not None,
514
- }
515
-
516
- @settings_app.post("/llm-config")
517
- def set_llm_config(update: LlmSettingsUpdate) -> dict[str, object]:
518
- """Persist the household's endpoint choice; applied on next start."""
519
- current = load_llm_config()
520
- if update.api_key is None:
521
- api_key = current.api_key # blank field: keep the stored key.
522
- elif update.api_key == "":
523
- api_key = None # explicit clear.
524
- else:
525
- api_key = update.api_key # a new key.
526
- new_config = LlmConfig(
527
- base_url=update.base_url.strip(),
528
- model=update.model.strip(),
529
- api_key=api_key,
530
- )
531
- write_llm_config(new_config)
532
- logger.info("LLM settings saved; restart ChittiOS to apply")
533
- return {
534
- "base_url": new_config.base_url,
535
- "model": new_config.model,
536
- "has_api_key": new_config.api_key is not None,
537
- "restart_required": True,
538
- }
539
-
540
- def _build_perception(
541
- self,
542
- store: EnrollmentStore,
543
- camera: CameraSource,
544
- mic: MicSource,
545
- detector: FaceDetectorProtocol,
546
- segmenter: SegmenterProtocol,
547
- face_embedder: SFaceEmbedder,
548
- voice_embedder: ECAPAEmbedder | None,
549
- ) -> PerceptionSeams:
550
- """Assemble the perception seams from the shared devices and models.
551
-
552
- The recognition providers are built from the roster loaded out of the
553
- store, so the household the robot recognises is exactly the one enrolled.
554
- The camera and microphone arrive already opened (or as silent null
555
- sources when a device could not be opened), so this only wires the
556
- pipelines around them. When no speaker embedder is present (the robot,
557
- where ECAPA does not fit) the voice provider degrades to a
558
- :class:`_NullVoiceProvider` so recognition runs on the face alone.
559
- """
560
- face_provider = FaceIdentityProvider(face_embedder, roster.face_roster(store))
561
- voice_provider: IdentityProvider[SpeechSegment] = (
562
- VoiceIdentityProvider(voice_embedder, roster.voice_roster(store))
563
- if voice_embedder is not None
564
- else _NullVoiceProvider()
565
- )
566
-
567
- return PerceptionSeams(
568
- face=FacePipeline(source=camera, detector=detector, provider=face_provider),
569
- mic=mic,
570
- segmenter=segmenter,
571
- voice_provider=voice_provider,
572
- )
573
-
574
- def _build_onboarding(
575
- self,
576
- *,
577
- store: EnrollmentStore,
578
- body: ReachyMiniAdapter,
579
- tts: TtsEngine,
580
- reachy_mini: ReachyMini,
581
- camera: CameraSource,
582
- mic: MicSource,
583
- detector: FaceDetectorProtocol,
584
- segmenter: SegmenterProtocol,
585
- face_embedder: SFaceEmbedder,
586
- voice_embedder: ECAPAEmbedder | None,
587
- stt: WhisperStt | None,
588
- stop_event: threading.Event,
589
- ) -> OwnerEnrollment:
590
- """The first-boot ritual, wired from the shared seams β€” or a friendly fallback.
591
-
592
- The whole spoken :class:`EnrollmentDialog` is assembled behind ``_guard``:
593
- if any of its concrete ports cannot be built β€” no STT model, no keychain,
594
- an unreadable device β€” the assembly degrades to :class:`_GreetAndIdleFallback`
595
- rather than raising, so an empty store never leaves the robot exiting in
596
- silence. The orchestrator drives whichever object comes back through the
597
- same ``enroll_owner`` seam.
598
- """
599
- speech_out = SyncSpeechOut(body, tts)
600
- dialog = _guard(
601
- "voice onboarding",
602
- lambda: self._assemble_dialog(
603
- store=store,
604
- speech_out=speech_out,
605
- reachy_mini=reachy_mini,
606
- camera=camera,
607
- mic=mic,
608
- detector=detector,
609
- segmenter=segmenter,
610
- face_embedder=face_embedder,
611
- voice_embedder=voice_embedder,
612
- stt=stt,
613
- stop_event=stop_event,
614
- ),
615
- )
616
- if dialog is not None:
617
- return dialog
618
-
619
- logger.info("voice onboarding is unavailable; the empty-store path will greet and idle")
620
- return _GreetAndIdleFallback(speech_out)
621
-
622
- def _assemble_dialog(
623
- self,
624
- *,
625
- store: EnrollmentStore,
626
- speech_out: SpeechOut,
627
- reachy_mini: ReachyMini,
628
- camera: CameraSource,
629
- mic: MicSource,
630
- detector: FaceDetectorProtocol,
631
- segmenter: SegmenterProtocol,
632
- face_embedder: SFaceEmbedder,
633
- voice_embedder: ECAPAEmbedder | None,
634
- stt: WhisperStt | None,
635
- stop_event: threading.Event,
636
- ) -> EnrollmentDialog:
637
- """Wire the concrete ports into the spoken enrollment dialog.
638
-
639
- Raises if any hard requirement is missing (STT is the one that can be
640
- absent on a fresh install); the caller guards this, turning that raise
641
- into the greet-and-idle fallback.
642
-
643
- Voice *enrollment* follows voice identification: with no speaker embedder
644
- (the robot) the sitting runs face-only β€” ``enroll_voice=False``, and the
645
- factory holds a never-called :class:`_NullVoiceEmbedder` so its wiring is
646
- uniform. Name and face capture, the antenna gate, and the passphrase
647
- custody are unchanged, so a household is still fully enrolled by voice on
648
- the robot; only the biometric *voice print* is omitted.
649
-
650
- The owner-hold wait is wired to ``stop_event`` (``should_abort``) so a
651
- daemon shutdown while the robot is waiting for the antenna hold is honoured
652
- promptly rather than only after the wait's timeout elapses.
653
- """
654
- if stt is None:
655
- raise RuntimeError("voice onboarding needs Whisper STT to hear names and answers")
656
-
657
- # On first boot the identity runtime does not exist yet, so recognition
658
- # reports a guest (no one recognised) β€” correct for the owner bootstrap,
659
- # which is gated on the antenna hold, not on a vouch.
660
- recognizer = SessionRecognizer(Session.guest)
661
- enroll_voice = voice_embedder is not None
662
- return EnrollmentDialog(
663
- speech_out=speech_out,
664
- speech_in=WhisperSpeechIn(mic, segmenter, stt),
665
- name_recorder=MicNameRecorder(mic, segmenter),
666
- recognizer=recognizer,
667
- gate=AuthorityGate(AntennaHoldSignal(reachy_mini), recognizer),
668
- conductor=CaptureConductor(speech_out),
669
- custody=PassphraseCustody(VAULT_PATH),
670
- store=store,
671
- face_source=PullFaceSource(camera, detector),
672
- voice_source=PullVoiceSource(mic, segmenter),
673
- enrollers=ModelEnrollerFactory(face_embedder, voice_embedder or _NullVoiceEmbedder()),
674
- should_abort=stop_event.is_set,
675
- enroll_voice=enroll_voice,
676
- # No spoken setup passphrase: it only fed the (fragile, STT-based)
677
- # recovery vault; the store is keyed by the on-device FileKeyStore.
678
- # Setup is antenna hold -> face + name -> done.
679
- require_passphrase=False,
680
- )
681
-
682
- def _build_tts(self) -> TtsEngine:
683
- """The speech engine β€” Piper when a voice model is present, else the stub.
684
-
685
- TTS is the one engine the orchestrator cannot do without: greeting has to
686
- have a voice. So a missing Piper model degrades to the deterministic tone
687
- rather than to ``None`` β€” a plain-spoken robot instead of a mute one.
688
- """
689
- model = next((p for p in (_BUNDLED_VOICE, _USER_VOICE) if p.exists()), None)
690
- if model is not None:
691
- engine = _guard(
692
- "Piper TTS",
693
- lambda: PiperTtsEngine(str(model), config_path=f"{model}.json"),
694
- )
695
- if engine is not None:
696
- return engine
697
- logger.info("no Piper voice model available; using the stub tone voice")
698
- return StubTtsEngine()
699
-
700
-
701
- if __name__ == "__main__":
702
- app = Chittios()
703
- try:
704
- app.wrapped_run()
705
- except KeyboardInterrupt:
706
- app.stop()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ChittiOS β€” the Reachy Mini app shell (Phase 7: the composed OS).
2
+
3
+ Phase 1 proved the publish β†’ install β†’ run pipeline with a greeting and idle
4
+ breathing. This is the real app: a thin shell that constructs the *concrete*,
5
+ hardware- and model-backed engines and hands them to ``chittios_core``'s
6
+ :class:`~chittios_core.orchestrator.ChittiOS`, which owns every line of composition
7
+ logic. Nothing here decides *what* ChittiOS does β€” it only supplies the real body,
8
+ the real camera and microphone, and the real STT/LLM/TTS to the seams the
9
+ orchestrator drives, exactly where a test supplies stubs.
10
+
11
+ Thin, and gracefully degrading
12
+ ------------------------------
13
+ Every heavy engine is optional: a fresh SBC may lack a Piper voice, a Whisper
14
+ model, or a reachable LLM, and the household should still get a robot that
15
+ recognises them and says hello rather than one that refuses to boot. So each
16
+ engine is built behind a small guarded factory that returns ``None`` (or a stub
17
+ fallback for the one engine β€” TTS β€” the greeting cannot do without) when its
18
+ dependency or device is absent. A missing LLM costs conversation and nothing
19
+ else; perception, greeting, and idle motion run regardless.
20
+
21
+ Idle breathing stays a raw-SDK control loop
22
+ --------------------------------------------
23
+ The "alive" breathing is the one thing that cannot go through the HAL motion
24
+ primitives: it needs ``set_target`` at a fixed 60 Hz (repeated ``goto_target``
25
+ each restart the interpolation β€” the judder Phase 1 diagnosed). So it stays here,
26
+ against the raw SDK, wrapped as an :class:`IdleMotion` the orchestrator runs on
27
+ its own thread β€” the reference control loop, unchanged, just handed to the
28
+ composition instead of being ``run``'s only body.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import logging
34
+ import os
35
+ import threading
36
+ import time
37
+ from collections.abc import Callable
38
+ from contextlib import ExitStack
39
+ from datetime import UTC, datetime
40
+ from pathlib import Path
41
+ from typing import TypeVar
42
+
43
+ import numpy as np
44
+ from fastapi import FastAPI
45
+ from numpy.typing import NDArray
46
+ from pydantic import BaseModel
47
+ from reachy_mini import ReachyMini, ReachyMiniApp
48
+ from reachy_mini.utils import create_head_pose
49
+
50
+ from chittios_core.context import ContextManager
51
+ from chittios_core.conversation.config import LlmConfig, load_llm_config, write_llm_config
52
+ from chittios_core.conversation.llm import OpenAICompatibleLlm
53
+ from chittios_core.conversation.stt import WhisperStt
54
+ from chittios_core.conversation.vad import OnnxVadSegmenter
55
+ from chittios_core.expression.config import VoiceConfig, load_voice_config, write_voice_config
56
+ from chittios_core.expression.elevenlabs import ElevenLabsTtsEngine
57
+ from chittios_core.expression.tts import (
58
+ FallbackTtsEngine,
59
+ PiperTtsEngine,
60
+ StubTtsEngine,
61
+ TtsEngine,
62
+ )
63
+ from chittios_core.identity import roster
64
+ from chittios_core.identity.face.detector import FaceDetectorProtocol, YuNetDetector
65
+ from chittios_core.identity.face.embedder import SFaceEmbedder
66
+ from chittios_core.identity.face.provider import FaceIdentityProvider
67
+ from chittios_core.identity.onboarding.adapters import (
68
+ AntennaHoldSignal,
69
+ MicNameRecorder,
70
+ ModelEnrollerFactory,
71
+ PassphraseCustody,
72
+ PullFaceSource,
73
+ PullVoiceSource,
74
+ SyncSpeechOut,
75
+ WhisperSpeechIn,
76
+ )
77
+ from chittios_core.identity.onboarding.authority import AuthorityGate
78
+ from chittios_core.identity.onboarding.conductor import CaptureConductor
79
+ from chittios_core.identity.onboarding.dialog import (
80
+ DialogOutcome,
81
+ DialogResult,
82
+ EnrollmentDialog,
83
+ )
84
+ from chittios_core.identity.onboarding.ports import SessionRecognizer, SpeechOut
85
+ from chittios_core.identity.session import Session
86
+ from chittios_core.identity.store.enrollments import EnrollmentStore
87
+ from chittios_core.identity.store.keys import (
88
+ FileKeyStore,
89
+ KeyCustodyError,
90
+ KeyringKeyStore,
91
+ get_or_create_key,
92
+ )
93
+ from chittios_core.identity.types import IdentityClaim, IdentityProvider, Modality
94
+ from chittios_core.identity.voice.embedder import ECAPAEmbedder
95
+ from chittios_core.identity.voice.provider import VoiceIdentityProvider
96
+ from chittios_core.identity.voice.segmenter import SegmenterProtocol, SpeechSegment
97
+ from chittios_core.orchestrator import (
98
+ ChittiOS,
99
+ ConversationSeams,
100
+ OwnerEnrollment,
101
+ PerceptionSeams,
102
+ )
103
+ from chittios_core.perception.loop import FacePipeline
104
+ from chittios_core.perception.sources import (
105
+ CameraSource,
106
+ MicSource,
107
+ ReachyMiniCameraSource,
108
+ SoundDeviceMicSource,
109
+ )
110
+ from hal.reachy_mini.adapter import ReachyMiniAdapter
111
+
112
+ logger = logging.getLogger(__name__)
113
+
114
+ _T = TypeVar("_T")
115
+
116
+ #: Where the enrolled household lives on the robot. The store creates its parent.
117
+ STORE_PATH = Path.home() / ".local" / "share" / "chittios" / "enrollments.db"
118
+
119
+ #: Where the recoverable passphrase custody vault lives β€” the Argon2id-wrapped
120
+ #: data key the recovery flow re-derives. Beside the store because a wrapped key
121
+ #: is ciphertext, useless without the household passphrase (see vault.py).
122
+ VAULT_PATH = Path.home() / ".local" / "share" / "chittios" / "passphrase.vault"
123
+
124
+ #: Piper voice model. Prefer the one bundled with the app so a fresh robot speaks
125
+ #: real words out of the box; fall back to a user-supplied model in the data dir.
126
+ _BUNDLED_VOICE = Path(__file__).parent / "assets" / "voice.onnx"
127
+ _USER_VOICE = Path.home() / ".local" / "share" / "chittios" / "voice.onnx"
128
+
129
+ #: Spoken by the empty-store fallback when the robot cannot run onboarding β€”
130
+ #: honest and warm rather than silent. The robot then idles (breathing) rather
131
+ #: than exiting, so a household that unboxes it into a half-provisioned state
132
+ #: still meets something alive.
133
+ ONBOARDING_UNAVAILABLE_GREETING = (
134
+ "Hello. I'm not quite ready to set up your household yet, but I'm here with you."
135
+ )
136
+
137
+ # ── idle breathing (the reference 60 Hz control loop) ──────────────────────
138
+
139
+ #: Control-loop rate. The fleet app uses 60 Hz; the docs floor smoothness at
140
+ #: 30 Hz. One ``set_target`` per tick, period held by adaptive sleep.
141
+ CONTROL_FREQUENCY_HZ = 60.0
142
+ CONTROL_PERIOD_S = 1.0 / CONTROL_FREQUENCY_HZ
143
+
144
+ #: Idle "breathing": the target pose is a smooth sine of time (not discrete
145
+ #: commands). Amplitudes kept small and slow, matching the reference breathing.
146
+ BREATHING_PITCH_DEG = 3.0
147
+ BREATHING_PITCH_HZ = 0.2
148
+ LOOK_YAW_DEG = 8.0
149
+ LOOK_YAW_HZ = 0.05
150
+
151
+
152
+ class BreathingMotion:
153
+ """The idle "alive" motion, as an :class:`IdleMotion` the orchestrator runs.
154
+
155
+ The Phase 1 control loop, unchanged in substance: one ``set_target`` per tick
156
+ at a monotonic-aligned 60 Hz, the pose a smooth sine of time. It lives against
157
+ the raw SDK rather than the HAL because it needs ``set_target``'s immediacy β€”
158
+ the HAL's queued ``goto_target`` would restart interpolation every tick and
159
+ judder. The orchestrator runs this on its own thread and stops it with the same
160
+ event that stops everything else.
161
+
162
+ Args:
163
+ mini: The connected robot to drive.
164
+ """
165
+
166
+ def __init__(self, mini: ReachyMini) -> None:
167
+ self._mini = mini
168
+
169
+ def run(self, stop: threading.Event) -> None:
170
+ """Breathe until ``stop`` is set, then return."""
171
+ start = time.monotonic()
172
+ while not stop.is_set():
173
+ tick_start = time.monotonic()
174
+ elapsed = tick_start - start
175
+
176
+ pitch_deg = BREATHING_PITCH_DEG * np.sin(2.0 * np.pi * BREATHING_PITCH_HZ * elapsed)
177
+ yaw_deg = LOOK_YAW_DEG * np.sin(2.0 * np.pi * LOOK_YAW_HZ * elapsed)
178
+ self._mini.set_target(head=create_head_pose(pitch=pitch_deg, yaw=yaw_deg, degrees=True))
179
+
180
+ # Adaptive sleep holds the period regardless of per-tick cost; the
181
+ # event doubles as an interruptible sleep, so a stop returns at once.
182
+ sleep_s = max(0.0, CONTROL_PERIOD_S - (time.monotonic() - tick_start))
183
+ if stop.wait(sleep_s):
184
+ break
185
+
186
+
187
+ class _GreetAndIdleFallback:
188
+ """The empty-store path when onboarding genuinely cannot be assembled.
189
+
190
+ An unboxed robot has an empty store and, normally, runs the spoken owner
191
+ enrollment. But if the pieces that ritual needs β€” a working microphone, the
192
+ voice models, the vault β€” cannot be built, the honest failure is not to exit
193
+ silently (a robot that does nothing looks broken) but to *say so* and stay
194
+ alive. Staying alive is the orchestrator's job now: it breathes on its own
195
+ thread and offers ``enroll_owner`` again on a cadence for the whole empty-store
196
+ lifetime, so this only owes a single greeting. It says its line the first time
197
+ it is offered and returns ``FAILED``; subsequent offers return ``FAILED``
198
+ silently, so the robot breathes on quietly rather than repeating itself.
199
+
200
+ It satisfies :class:`OwnerEnrollment` structurally, so the orchestrator drives
201
+ it through exactly the same ``enroll_owner`` seam it drives the real dialog
202
+ through β€” the orchestrator's own empty-store semantics are untouched; only
203
+ *what* is wired into that seam differs when the dialog could not be built.
204
+
205
+ Args:
206
+ speech_out: How the greeting is spoken (the same sync TTS seam the dialog
207
+ would have used).
208
+ greeting: The single line spoken on the first offer.
209
+ """
210
+
211
+ def __init__(
212
+ self,
213
+ speech_out: SpeechOut,
214
+ *,
215
+ greeting: str = ONBOARDING_UNAVAILABLE_GREETING,
216
+ ) -> None:
217
+ self._speech_out = speech_out
218
+ self._greeting = greeting
219
+ self._greeted = False
220
+
221
+ def enroll_owner(self) -> DialogResult:
222
+ """Greet on the first offer, then stay quiet. Nothing is ever enrolled."""
223
+ if not self._greeted:
224
+ self._greeted = True
225
+ try:
226
+ self._speech_out.speak(self._greeting)
227
+ except Exception:
228
+ logger.warning("the fallback greeting could not be spoken", exc_info=True)
229
+ return DialogResult(DialogOutcome.FAILED)
230
+
231
+
232
+ # ── null sources (degrade when a device is absent) ─────────────────────────
233
+
234
+
235
+ class _NullCamera:
236
+ """A :class:`CameraSource` that always sees nothing.
237
+
238
+ Used when no camera can be opened: perception then runs on voice alone rather
239
+ than failing to start. A returned ``None`` is the source contract's ordinary
240
+ "nothing right now", so the loop simply never gets a face.
241
+ """
242
+
243
+ def read(self) -> None:
244
+ return None
245
+
246
+
247
+ class _NullMic:
248
+ """A :class:`MicSource` that always hears nothing β€” the audio counterpart."""
249
+
250
+ def read(self) -> None:
251
+ return None
252
+
253
+
254
+ class _NullVoiceProvider:
255
+ """A voice :class:`IdentityProvider` that recognises no one β€” face-only builds.
256
+
257
+ The speaker encoder (ECAPA) is torch- and speechbrain-backed, ~2 GB that does
258
+ not fit on the Reachy Mini, so voice **identification** is dropped on the
259
+ robot (the owner's call). Perception still segments audio β€” the conversation
260
+ loop needs those utterances for STT β€” but the voice *provider* it drives is
261
+ this: every utterance returns an unknown claim, exactly as the real provider
262
+ does for anyone unenrolled, so the fusion and authorization paths downstream
263
+ are unchanged and simply never receive a voice match. Recognition runs on the
264
+ face alone.
265
+
266
+ It satisfies :class:`~chittios_core.identity.types.IdentityProvider`\\
267
+ ``[SpeechSegment]`` structurally, so it drops into the same perception seam
268
+ the real provider fills, and it constructs nothing torch-backed.
269
+ """
270
+
271
+ @property
272
+ def modality(self) -> Modality:
273
+ return Modality.VOICE
274
+
275
+ def identify(
276
+ self, observation: SpeechSegment, /, *, observed_at: datetime | None = None
277
+ ) -> IdentityClaim:
278
+ """Return an unknown voice claim for any utterance β€” no one is identified."""
279
+ return IdentityClaim.unknown(
280
+ modality=Modality.VOICE, observed_at=observed_at or datetime.now(UTC)
281
+ )
282
+
283
+
284
+ class _NullVoiceEmbedder:
285
+ """A voice embedder that is never called β€” the placeholder for face-only sittings.
286
+
287
+ ``ModelEnrollerFactory`` holds a voice embedder so it can mint voice enrollers,
288
+ but a face-only enrollment (``enroll_voice=False``) never asks it for one, so
289
+ this stand-in is only ever *held*, never *invoked*. It exists so the factory's
290
+ wiring is uniform whether or not a speaker encoder is present, without
291
+ constructing the torch-backed ECAPA on a robot that cannot run it. Being
292
+ called would mean a face-only sitting tried to embed voice after all, which is
293
+ a wiring bug worth failing loudly on rather than papering over with a vector.
294
+ """
295
+
296
+ def embed(self, canonical_audio: NDArray[np.float32], /) -> NDArray[np.float32]:
297
+ raise NotImplementedError(
298
+ "voice enrollment is disabled on this build; _NullVoiceEmbedder must never be called"
299
+ )
300
+
301
+
302
+ def _guard(what: str, build: Callable[[], _T]) -> _T | None:
303
+ """Build an optional engine, degrading to ``None`` if its runtime is absent.
304
+
305
+ The one place the "a missing model must not stop the robot booting" rule is
306
+ enforced: any failure to construct β€” a missing package, an absent model file,
307
+ an unopenable device β€” is logged and turned into ``None`` rather than raised.
308
+ """
309
+ try:
310
+ return build()
311
+ except Exception:
312
+ logger.warning("%s is unavailable; ChittiOS will run without it", what, exc_info=True)
313
+ return None
314
+
315
+
316
+ def _resolve_store_key() -> bytes:
317
+ """The enrollment store's encryption key: OS keychain where one exists, else a
318
+ mode-0600 key file on a headless Linux host.
319
+
320
+ The robot is headless Linux with no login keyring, so ``KeyringKeyStore``
321
+ raises there. ``FileKeyStore`` keeps the key in a 0600 file beside the store β€”
322
+ weaker than a keychain (the key sits near the data it protects), but on a box
323
+ with no keychain it is that or no at-rest encryption at all. Windows and macOS
324
+ keep the keychain; ``FileKeyStore`` is POSIX-only, so this only falls back on
325
+ a POSIX host.
326
+ """
327
+ try:
328
+ return get_or_create_key(KeyringKeyStore())
329
+ except KeyCustodyError:
330
+ if os.name != "posix":
331
+ raise
332
+ key_dir = STORE_PATH.parent
333
+ key_dir.mkdir(parents=True, exist_ok=True)
334
+ logger.warning("no OS keychain available; using a 0600 key file in %s", key_dir)
335
+ return get_or_create_key(FileKeyStore(key_dir))
336
+
337
+
338
+ class LlmSettingsUpdate(BaseModel):
339
+ """The dashboard's ``POST /llm-config`` body β€” the three settings, plus key semantics.
340
+
341
+ ``api_key`` distinguishes three intents a single password field cannot
342
+ otherwise express: ``None`` (field left blank) leaves the stored key
343
+ untouched, an empty string clears it, and any other string sets it β€” so
344
+ changing the URL never silently drops a previously saved key.
345
+
346
+ Defined at module scope, not inside :meth:`Chittios._register_settings_routes`,
347
+ on purpose: this module uses ``from __future__ import annotations`` (PEP 563),
348
+ so the route handler's ``update: LlmSettingsUpdate`` annotation is a *string*
349
+ that FastAPI resolves with :func:`typing.get_type_hints` against the module's
350
+ globals. A method-local class is not in those globals, so the annotation
351
+ fails to resolve to this Pydantic model and FastAPI falls back to treating
352
+ ``update`` as a required *query* parameter β€” which no JSON body can satisfy,
353
+ yielding ``422 Unprocessable Entity`` on every save. Keeping the model here,
354
+ where ``get_type_hints`` can find it, is what makes the body bind.
355
+ """
356
+
357
+ base_url: str
358
+ model: str
359
+ api_key: str | None = None
360
+
361
+
362
+ class VoiceSettingsUpdate(BaseModel):
363
+ """The dashboard's ``POST /voice-config`` body β€” just the ElevenLabs key.
364
+
365
+ The same three-way ``api_key`` contract as :class:`LlmSettingsUpdate`: ``None``
366
+ (a blank field) keeps the stored key, ``""`` clears it, and any other string
367
+ sets it β€” so a household never loses a saved key by revisiting the page. The
368
+ voice and model are not surfaced in the UI (a warm premade voice is the
369
+ default), so they are not part of this body. Defined at module scope for the
370
+ same PEP 563 reason spelled out on :class:`LlmSettingsUpdate`: FastAPI must be
371
+ able to resolve the annotation against the module globals for the JSON body to
372
+ bind instead of degrading to a 422.
373
+ """
374
+
375
+ api_key: str | None = None
376
+
377
+
378
+ class Chittios(ReachyMiniApp): # type: ignore[misc] # reachy_mini ships no stubs; base resolves to Any
379
+ # A small settings web UI, served by the daemon on this URL, lets a household
380
+ # point ChittiOS at their own LLM endpoint from the robot dashboard. The base
381
+ # class builds ``self.settings_app`` (a FastAPI app) from this URL; we add the
382
+ # config routes in ``run``. ``0.0.0.0`` so the dashboard reaches it over the LAN.
383
+ custom_app_url: str | None = "http://0.0.0.0:8042"
384
+ request_media_backend: str | None = None
385
+
386
+ def __init__(self, running_on_wireless: bool = False) -> None:
387
+ """Build the app and register the settings routes before the server serves.
388
+
389
+ The base class builds ``self.settings_app`` here and ``wrapped_run`` starts
390
+ its web-server thread *before* it calls :meth:`run`. Registering the config
391
+ routes inside ``run`` (as this once did) therefore leaves a window β€” the
392
+ whole time the robot spends connecting to the daemon before ``run`` begins β€”
393
+ during which the server is already accepting requests but ``/llm-config``
394
+ does not exist yet, so the dashboard's on-load fetch races ahead of the
395
+ route and gets a 404. Registering them now, at construction, closes that
396
+ window: the routes are on the app before the first request can arrive.
397
+ """
398
+ super().__init__(running_on_wireless)
399
+ self._register_settings_routes()
400
+
401
+ def run(self, reachy_mini: ReachyMini, stop_event: threading.Event) -> None:
402
+ """Construct the real engines and hand them to the orchestrator.
403
+
404
+ Opens the camera and microphone for the app's lifetime (released on
405
+ return), builds whatever recognition, conversation, and speech engines the
406
+ machine can provide, and lets :class:`ChittiOS` compose and run them until
407
+ the daemon sets ``stop_event``.
408
+ """
409
+ logging.basicConfig(level=logging.INFO)
410
+ body = ReachyMiniAdapter(reachy_mini)
411
+ store = EnrollmentStore(STORE_PATH, _resolve_store_key())
412
+ tts = self._build_tts()
413
+
414
+ with ExitStack() as sources:
415
+ # The camera and microphone are shared: on an empty store only
416
+ # onboarding reads them, on a populated one only perception does, and
417
+ # the two paths never run together β€” so one source each, not two
418
+ # fighting over an exclusive device.
419
+ #
420
+ # The camera comes from the SDK, not OpenCV: on the robot the daemon
421
+ # owns the Orbbec and streams frames over a GStreamer IPC endpoint, so
422
+ # a direct ``cv2.VideoCapture(0)`` fails ("could not open camera 0").
423
+ # ``reachy_mini.media.get_frame()`` is how an app gets frames while the
424
+ # daemon holds the device, and it already returns exactly the BGR
425
+ # frames the perception loop expects.
426
+ camera: CameraSource = (
427
+ _guard("camera", lambda: ReachyMiniCameraSource(reachy_mini.media)) or _NullCamera()
428
+ )
429
+ mic: MicSource = (
430
+ _guard("microphone", lambda: sources.enter_context(SoundDeviceMicSource()))
431
+ or _NullMic()
432
+ )
433
+ # The heavy recognition models, built once and shared by perception and
434
+ # onboarding so neither boot path pays to load them twice.
435
+ detector = YuNetDetector()
436
+ segmenter = OnnxVadSegmenter()
437
+ face_embedder = SFaceEmbedder()
438
+ # Voice identification is the one torch-backed engine left on this
439
+ # path: ECAPA pulls speechbrain and torch (~2 GB), which do not fit the
440
+ # Reachy Mini. Behind _guard it degrades to None when they are absent β€”
441
+ # recognition then runs on the face alone (_NullVoiceProvider) and the
442
+ # first-boot sitting enrols face-only. On a dev box with the `voice`
443
+ # extra installed it builds as before and voice-ID is fully live.
444
+ voice_embedder = _guard("voice identification (ECAPA)", ECAPAEmbedder)
445
+ # ``tiny.en`` rather than ``base``: on this SBC's CPU, ``base`` spends
446
+ # ~12 s decoding a 2 s turn, so utterances queue faster than they clear
447
+ # and a reply lands a minute late β€” unusable. The English-only ``tiny``
448
+ # is several times faster for a modest accuracy cost that short
449
+ # household turns tolerate well, keeping time-to-answer conversational.
450
+ #
451
+ # The ``.en`` build also *is* the language lock: forcing English keeps
452
+ # faster-whisper from auto-detecting per utterance, which on short,
453
+ # quiet turns β€” a name, a yes β€” has mis-fired to Italian on the robot
454
+ # and garbled every transcript. ``language="en"`` is kept as belt and
455
+ # braces. The core WhisperStt default stays language-agnostic; only the
456
+ # app pins the model and the language ChittiOS speaks.
457
+ stt = _guard("Whisper STT", lambda: WhisperStt("tiny.en", language="en"))
458
+
459
+ perception = self._build_perception(
460
+ store, camera, mic, detector, segmenter, face_embedder, voice_embedder
461
+ )
462
+ conversation = ConversationSeams(stt=stt, llm=_guard("LLM", self._build_llm))
463
+ onboarding = self._build_onboarding(
464
+ store=store,
465
+ body=body,
466
+ tts=tts,
467
+ reachy_mini=reachy_mini,
468
+ camera=camera,
469
+ mic=mic,
470
+ detector=detector,
471
+ segmenter=segmenter,
472
+ face_embedder=face_embedder,
473
+ voice_embedder=voice_embedder,
474
+ stt=stt,
475
+ stop_event=stop_event,
476
+ )
477
+
478
+ orchestrator = ChittiOS(
479
+ store=store,
480
+ perception=perception,
481
+ conversation=conversation,
482
+ tts=tts,
483
+ context=ContextManager(),
484
+ idle_motion=BreathingMotion(reachy_mini),
485
+ onboarding=onboarding,
486
+ )
487
+ orchestrator.run(body, stop_event=stop_event)
488
+
489
+ body.close()
490
+
491
+ def _build_llm(self) -> OpenAICompatibleLlm:
492
+ """Construct the conversation LLM from the household's resolved config.
493
+
494
+ The endpoint is not hardcoded: :func:`load_llm_config` resolves it from
495
+ the environment, then the config file the settings UI writes, then the
496
+ built-in default (the Spark's Gemma). Whatever comes back is where this
497
+ robot sends conversation transcripts β€” local by default, or wherever the
498
+ household pointed it. Built behind ``_guard`` in :meth:`run`, so a
499
+ malformed config or missing HTTP stack costs conversation and nothing else.
500
+ """
501
+ config = load_llm_config()
502
+ logger.info("LLM endpoint: %s (model %s)", config.base_url, config.model)
503
+ return OpenAICompatibleLlm(
504
+ base_url=config.base_url,
505
+ model=config.model,
506
+ api_key=config.api_key,
507
+ )
508
+
509
+ def _register_settings_routes(self) -> None:
510
+ """Add the LLM-endpoint settings routes to the dashboard's settings app.
511
+
512
+ The base class builds ``self.settings_app`` from ``custom_app_url`` and
513
+ serves ``static/index.html`` as the dashboard page; here we give that page
514
+ an API. ``GET /llm-config`` reports the resolved endpoint and whether a key
515
+ is set (never the key itself β€” a secret is not echoed back over HTTP);
516
+ ``POST /llm-config`` writes the household's choice to the config file.
517
+
518
+ Writes take effect on the next app start β€” there is no live client swap, by
519
+ design: rebuilding the conversation engine mid-turn is far more surface
520
+ area than the feature needs, so the contract is simply "restart to apply".
521
+ """
522
+ # The base class exposes ``settings_app`` untyped (reachy_mini ships no
523
+ # stubs); bind it to a concrete ``FastAPI`` local so its route decorators
524
+ # are typed rather than ``Any`` (which mypy rejects as an untyped decorator).
525
+ settings_app: FastAPI | None = self.settings_app
526
+ if settings_app is None: # webserver disabled; nothing to wire.
527
+ return
528
+
529
+ @settings_app.get("/llm-config")
530
+ def get_llm_config() -> dict[str, object]:
531
+ """Report the resolved endpoint for the dashboard to display."""
532
+ config = load_llm_config()
533
+ return {
534
+ "base_url": config.base_url,
535
+ "model": config.model,
536
+ "has_api_key": config.api_key is not None,
537
+ }
538
+
539
+ @settings_app.post("/llm-config")
540
+ def set_llm_config(update: LlmSettingsUpdate) -> dict[str, object]:
541
+ """Persist the household's endpoint choice; applied on next start."""
542
+ current = load_llm_config()
543
+ if update.api_key is None:
544
+ api_key = current.api_key # blank field: keep the stored key.
545
+ elif update.api_key == "":
546
+ api_key = None # explicit clear.
547
+ else:
548
+ api_key = update.api_key # a new key.
549
+ new_config = LlmConfig(
550
+ base_url=update.base_url.strip(),
551
+ model=update.model.strip(),
552
+ api_key=api_key,
553
+ )
554
+ write_llm_config(new_config)
555
+ logger.info("LLM settings saved; restart ChittiOS to apply")
556
+ return {
557
+ "base_url": new_config.base_url,
558
+ "model": new_config.model,
559
+ "has_api_key": new_config.api_key is not None,
560
+ "restart_required": True,
561
+ }
562
+
563
+ @settings_app.get("/voice-config")
564
+ def get_voice_config() -> dict[str, object]:
565
+ """Report whether a cloud voice is configured β€” never the key itself.
566
+
567
+ The dashboard uses this to decide whether to show the key box at all:
568
+ once a key is stored it hides the field, so onboarding asks once and
569
+ never again.
570
+ """
571
+ config = load_voice_config()
572
+ return {"has_api_key": config.api_key is not None}
573
+
574
+ @settings_app.post("/voice-config")
575
+ def set_voice_config(update: VoiceSettingsUpdate) -> dict[str, object]:
576
+ """Persist the household's ElevenLabs key; applied on next start."""
577
+ current = load_voice_config()
578
+ if update.api_key is None:
579
+ api_key = current.api_key # blank field: keep the stored key.
580
+ elif update.api_key.strip() == "":
581
+ api_key = None # explicit clear (empty, or whitespace only).
582
+ else:
583
+ api_key = update.api_key.strip() # a new key.
584
+ new_config = VoiceConfig(
585
+ api_key=api_key,
586
+ voice_id=current.voice_id,
587
+ model_id=current.model_id,
588
+ )
589
+ write_voice_config(new_config)
590
+ logger.info("voice settings saved; restart ChittiOS to apply")
591
+ return {"has_api_key": new_config.api_key is not None, "restart_required": True}
592
+
593
+ def _build_perception(
594
+ self,
595
+ store: EnrollmentStore,
596
+ camera: CameraSource,
597
+ mic: MicSource,
598
+ detector: FaceDetectorProtocol,
599
+ segmenter: SegmenterProtocol,
600
+ face_embedder: SFaceEmbedder,
601
+ voice_embedder: ECAPAEmbedder | None,
602
+ ) -> PerceptionSeams:
603
+ """Assemble the perception seams from the shared devices and models.
604
+
605
+ The recognition providers are built from the roster loaded out of the
606
+ store, so the household the robot recognises is exactly the one enrolled.
607
+ The camera and microphone arrive already opened (or as silent null
608
+ sources when a device could not be opened), so this only wires the
609
+ pipelines around them. When no speaker embedder is present (the robot,
610
+ where ECAPA does not fit) the voice provider degrades to a
611
+ :class:`_NullVoiceProvider` so recognition runs on the face alone.
612
+ """
613
+ face_provider = FaceIdentityProvider(face_embedder, roster.face_roster(store))
614
+ voice_provider: IdentityProvider[SpeechSegment] = (
615
+ VoiceIdentityProvider(voice_embedder, roster.voice_roster(store))
616
+ if voice_embedder is not None
617
+ else _NullVoiceProvider()
618
+ )
619
+
620
+ return PerceptionSeams(
621
+ face=FacePipeline(source=camera, detector=detector, provider=face_provider),
622
+ mic=mic,
623
+ segmenter=segmenter,
624
+ voice_provider=voice_provider,
625
+ )
626
+
627
+ def _build_onboarding(
628
+ self,
629
+ *,
630
+ store: EnrollmentStore,
631
+ body: ReachyMiniAdapter,
632
+ tts: TtsEngine,
633
+ reachy_mini: ReachyMini,
634
+ camera: CameraSource,
635
+ mic: MicSource,
636
+ detector: FaceDetectorProtocol,
637
+ segmenter: SegmenterProtocol,
638
+ face_embedder: SFaceEmbedder,
639
+ voice_embedder: ECAPAEmbedder | None,
640
+ stt: WhisperStt | None,
641
+ stop_event: threading.Event,
642
+ ) -> OwnerEnrollment:
643
+ """The first-boot ritual, wired from the shared seams β€” or a friendly fallback.
644
+
645
+ The whole spoken :class:`EnrollmentDialog` is assembled behind ``_guard``:
646
+ if any of its concrete ports cannot be built β€” no STT model, no keychain,
647
+ an unreadable device β€” the assembly degrades to :class:`_GreetAndIdleFallback`
648
+ rather than raising, so an empty store never leaves the robot exiting in
649
+ silence. The orchestrator drives whichever object comes back through the
650
+ same ``enroll_owner`` seam.
651
+ """
652
+ speech_out = SyncSpeechOut(body, tts)
653
+ dialog = _guard(
654
+ "voice onboarding",
655
+ lambda: self._assemble_dialog(
656
+ store=store,
657
+ speech_out=speech_out,
658
+ reachy_mini=reachy_mini,
659
+ camera=camera,
660
+ mic=mic,
661
+ detector=detector,
662
+ segmenter=segmenter,
663
+ face_embedder=face_embedder,
664
+ voice_embedder=voice_embedder,
665
+ stt=stt,
666
+ stop_event=stop_event,
667
+ ),
668
+ )
669
+ if dialog is not None:
670
+ return dialog
671
+
672
+ logger.info("voice onboarding is unavailable; the empty-store path will greet and idle")
673
+ return _GreetAndIdleFallback(speech_out)
674
+
675
+ def _assemble_dialog(
676
+ self,
677
+ *,
678
+ store: EnrollmentStore,
679
+ speech_out: SpeechOut,
680
+ reachy_mini: ReachyMini,
681
+ camera: CameraSource,
682
+ mic: MicSource,
683
+ detector: FaceDetectorProtocol,
684
+ segmenter: SegmenterProtocol,
685
+ face_embedder: SFaceEmbedder,
686
+ voice_embedder: ECAPAEmbedder | None,
687
+ stt: WhisperStt | None,
688
+ stop_event: threading.Event,
689
+ ) -> EnrollmentDialog:
690
+ """Wire the concrete ports into the spoken enrollment dialog.
691
+
692
+ Raises if any hard requirement is missing (STT is the one that can be
693
+ absent on a fresh install); the caller guards this, turning that raise
694
+ into the greet-and-idle fallback.
695
+
696
+ Voice *enrollment* follows voice identification: with no speaker embedder
697
+ (the robot) the sitting runs face-only β€” ``enroll_voice=False``, and the
698
+ factory holds a never-called :class:`_NullVoiceEmbedder` so its wiring is
699
+ uniform. Name and face capture, the antenna gate, and the passphrase
700
+ custody are unchanged, so a household is still fully enrolled by voice on
701
+ the robot; only the biometric *voice print* is omitted.
702
+
703
+ The owner-hold wait is wired to ``stop_event`` (``should_abort``) so a
704
+ daemon shutdown while the robot is waiting for the antenna hold is honoured
705
+ promptly rather than only after the wait's timeout elapses.
706
+ """
707
+ if stt is None:
708
+ raise RuntimeError("voice onboarding needs Whisper STT to hear names and answers")
709
+
710
+ # On first boot the identity runtime does not exist yet, so recognition
711
+ # reports a guest (no one recognised) β€” correct for the owner bootstrap,
712
+ # which is gated on the antenna hold, not on a vouch.
713
+ recognizer = SessionRecognizer(Session.guest)
714
+ enroll_voice = voice_embedder is not None
715
+ return EnrollmentDialog(
716
+ speech_out=speech_out,
717
+ speech_in=WhisperSpeechIn(mic, segmenter, stt),
718
+ name_recorder=MicNameRecorder(mic, segmenter),
719
+ recognizer=recognizer,
720
+ gate=AuthorityGate(AntennaHoldSignal(reachy_mini), recognizer),
721
+ conductor=CaptureConductor(speech_out),
722
+ custody=PassphraseCustody(VAULT_PATH),
723
+ store=store,
724
+ face_source=PullFaceSource(camera, detector),
725
+ voice_source=PullVoiceSource(mic, segmenter),
726
+ enrollers=ModelEnrollerFactory(face_embedder, voice_embedder or _NullVoiceEmbedder()),
727
+ should_abort=stop_event.is_set,
728
+ enroll_voice=enroll_voice,
729
+ # No spoken setup passphrase: it only fed the (fragile, STT-based)
730
+ # recovery vault; the store is keyed by the on-device FileKeyStore.
731
+ # Setup is antenna hold -> face + name -> done.
732
+ require_passphrase=False,
733
+ )
734
+
735
+ def _build_tts(self) -> TtsEngine:
736
+ """The speech engine: the household's cloud voice when configured, else local.
737
+
738
+ A configured ElevenLabs key (from ``ELEVENLABS_API_KEY`` or the settings
739
+ file the onboarding box writes) builds the cloud engine, wrapped so a
740
+ failure β€” a bad key, an exhausted quota, a dropped connection β€” falls back
741
+ to the local voice rather than muting the robot. No key means the local
742
+ voice outright. The local engine is always built first, precisely so it is
743
+ there to be the fallback.
744
+ """
745
+ local = self._build_local_tts()
746
+ voice = load_voice_config()
747
+ key = voice.api_key
748
+ if key is not None:
749
+ cloud = _guard(
750
+ "ElevenLabs TTS",
751
+ lambda: ElevenLabsTtsEngine(
752
+ key, voice_id=voice.voice_id, model_id=voice.model_id
753
+ ),
754
+ )
755
+ if cloud is not None:
756
+ logger.info(
757
+ "ElevenLabs voice enabled (voice %s, model %s); local voice is the fallback",
758
+ voice.voice_id,
759
+ voice.model_id,
760
+ )
761
+ return FallbackTtsEngine(cloud, fallback=local)
762
+ return local
763
+
764
+ def _build_local_tts(self) -> TtsEngine:
765
+ """The local speech engine β€” Piper when a voice model is present, else the stub.
766
+
767
+ TTS is the one engine the orchestrator cannot do without: greeting has to
768
+ have a voice. So a missing Piper model degrades to the deterministic tone
769
+ rather than to ``None`` β€” a plain-spoken robot instead of a mute one. This
770
+ is also the fallback the cloud voice degrades to, so it must never itself
771
+ return ``None``.
772
+ """
773
+ model = next((p for p in (_BUNDLED_VOICE, _USER_VOICE) if p.exists()), None)
774
+ if model is not None:
775
+ engine = _guard(
776
+ "Piper TTS",
777
+ lambda: PiperTtsEngine(str(model), config_path=f"{model}.json"),
778
+ )
779
+ if engine is not None:
780
+ return engine
781
+ logger.info("no Piper voice model available; using the stub tone voice")
782
+ return StubTtsEngine()
783
+
784
+
785
+ if __name__ == "__main__":
786
+ app = Chittios()
787
+ try:
788
+ app.wrapped_run()
789
+ except KeyboardInterrupt:
790
+ app.stop()
chittios/static/index.html CHANGED
@@ -46,6 +46,22 @@
46
  </label>
47
  </label>
48
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
  <div class="actions">
50
  <button id="save-btn" type="submit">Save settings</button>
51
  <span id="status" class="status" role="status" aria-live="polite"></span>
 
46
  </label>
47
  </label>
48
 
49
+ <div class="voice" id="voice-section">
50
+ <span class="label">Premium voice <span class="opt">optional</span></span>
51
+ <div id="voice-input-wrap">
52
+ <input id="eleven-key" name="eleven_key" type="password" autocomplete="off"
53
+ placeholder="Paste your ElevenLabs API key">
54
+ <span class="hint">Gives ChittiOS a natural cloud voice (ElevenLabs) instead of the
55
+ built-in one. Only the robot's <em>spoken reply</em> is sent for synthesis β€”
56
+ never your family's microphone audio, which always stays on the device. Leave
57
+ blank to keep the local voice.</span>
58
+ </div>
59
+ <p class="voice-on" id="voice-enabled" hidden>
60
+ βœ“ ElevenLabs voice enabled.
61
+ <button type="button" id="replace-voice" class="linkish">Replace key</button>
62
+ </p>
63
+ </div>
64
+
65
  <div class="actions">
66
  <button id="save-btn" type="submit">Save settings</button>
67
  <span id="status" class="status" role="status" aria-live="polite"></span>
chittios/static/main.js CHANGED
@@ -1,10 +1,13 @@
1
- // The ChittiOS settings page: read the current LLM endpoint, let the household
2
- // change it, and write it back. The API-key field is never pre-filled β€” the
3
- // server reports only whether a key is set, and a blank field leaves it as-is
4
- // (an explicit "Remove" checkbox clears it), so editing the URL can never
5
- // silently drop a saved key.
 
 
6
 
7
  const CONFIG_ENDPOINT = "/llm-config";
 
8
 
9
  const els = {
10
  form: document.getElementById("llm-form"),
@@ -15,6 +18,10 @@ const els = {
15
  keyHint: document.getElementById("key-hint"),
16
  saveBtn: document.getElementById("save-btn"),
17
  status: document.getElementById("status"),
 
 
 
 
18
  };
19
 
20
  function setStatus(message, kind) {
@@ -22,6 +29,8 @@ function setStatus(message, kind) {
22
  els.status.className = kind ? `status ${kind}` : "status";
23
  }
24
 
 
 
25
  // Reflect whether a key is currently stored, without ever revealing it.
26
  function reflectKeyState(hasApiKey) {
27
  els.apiKey.placeholder = hasApiKey
@@ -56,6 +65,48 @@ function resolveApiKey() {
56
  return typed.length > 0 ? typed : null;
57
  }
58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59
  async function saveConfig(event) {
60
  event.preventDefault();
61
 
@@ -79,14 +130,23 @@ async function saveConfig(event) {
79
  reflectKeyState(Boolean(data.has_api_key));
80
  els.apiKey.value = "";
81
  els.clearKey.checked = false;
 
 
 
82
  setStatus("Saved. Restart ChittiOS to apply.", "ok");
83
  } catch (e) {
84
  setStatus("Could not save settings.", "err");
85
- console.error("Failed to save LLM config:", e);
86
  } finally {
87
  els.saveBtn.disabled = false;
88
  }
89
  }
90
 
91
  els.form.addEventListener("submit", saveConfig);
 
 
 
 
 
92
  loadConfig();
 
 
1
+ // The ChittiOS settings page: read the current LLM endpoint and (optionally) an
2
+ // ElevenLabs voice key, let the household change them, and write them back.
3
+ //
4
+ // Neither secret is ever pre-filled. The server reports only *whether* each key
5
+ // is set. The LLM key field stays visible so it can be changed; the ElevenLabs
6
+ // key box, by contrast, is shown until a key is stored and then hidden β€” onboard
7
+ // once, don't ask again β€” with a "Replace key" link for the rare rotation.
8
 
9
  const CONFIG_ENDPOINT = "/llm-config";
10
+ const VOICE_ENDPOINT = "/voice-config";
11
 
12
  const els = {
13
  form: document.getElementById("llm-form"),
 
18
  keyHint: document.getElementById("key-hint"),
19
  saveBtn: document.getElementById("save-btn"),
20
  status: document.getElementById("status"),
21
+ elevenKey: document.getElementById("eleven-key"),
22
+ voiceInputWrap: document.getElementById("voice-input-wrap"),
23
+ voiceEnabled: document.getElementById("voice-enabled"),
24
+ replaceVoice: document.getElementById("replace-voice"),
25
  };
26
 
27
  function setStatus(message, kind) {
 
29
  els.status.className = kind ? `status ${kind}` : "status";
30
  }
31
 
32
+ // ── LLM endpoint ─────────────────────────────────────────────────────────
33
+
34
  // Reflect whether a key is currently stored, without ever revealing it.
35
  function reflectKeyState(hasApiKey) {
36
  els.apiKey.placeholder = hasApiKey
 
65
  return typed.length > 0 ? typed : null;
66
  }
67
 
68
+ // ── ElevenLabs voice ─────────────────────────────────────────────────────
69
+
70
+ // Show the key box only until a key is stored; after that, show the confirmation
71
+ // and hide the field β€” the "onboard once, don't ask again" behaviour.
72
+ function reflectVoiceState(hasKey) {
73
+ els.voiceInputWrap.hidden = hasKey;
74
+ els.voiceEnabled.hidden = !hasKey;
75
+ if (hasKey) els.elevenKey.value = "";
76
+ }
77
+
78
+ async function loadVoiceConfig() {
79
+ try {
80
+ const resp = await fetch(VOICE_ENDPOINT);
81
+ if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
82
+ const data = await resp.json();
83
+ reflectVoiceState(Boolean(data.has_api_key));
84
+ } catch (e) {
85
+ // A missing voice route (older build) just means no cloud voice β€” leave
86
+ // the box shown rather than blocking the page on it.
87
+ console.error("Failed to load voice config:", e);
88
+ }
89
+ }
90
+
91
+ // Save the ElevenLabs key only when the box is showing (onboarding or replace)
92
+ // and something was actually typed; otherwise there is nothing to do.
93
+ async function saveVoiceKey() {
94
+ if (els.voiceInputWrap.hidden) return;
95
+ const typed = els.elevenKey.value.trim();
96
+ if (!typed) return;
97
+
98
+ const resp = await fetch(VOICE_ENDPOINT, {
99
+ method: "POST",
100
+ headers: { "Content-Type": "application/json" },
101
+ body: JSON.stringify({ api_key: typed }),
102
+ });
103
+ if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
104
+ const data = await resp.json();
105
+ reflectVoiceState(Boolean(data.has_api_key));
106
+ }
107
+
108
+ // ── save ─────────────────────────────────────────────────────────────────
109
+
110
  async function saveConfig(event) {
111
  event.preventDefault();
112
 
 
130
  reflectKeyState(Boolean(data.has_api_key));
131
  els.apiKey.value = "";
132
  els.clearKey.checked = false;
133
+
134
+ await saveVoiceKey();
135
+
136
  setStatus("Saved. Restart ChittiOS to apply.", "ok");
137
  } catch (e) {
138
  setStatus("Could not save settings.", "err");
139
+ console.error("Failed to save settings:", e);
140
  } finally {
141
  els.saveBtn.disabled = false;
142
  }
143
  }
144
 
145
  els.form.addEventListener("submit", saveConfig);
146
+ els.replaceVoice.addEventListener("click", () => {
147
+ reflectVoiceState(false);
148
+ els.elevenKey.focus();
149
+ });
150
+
151
  loadConfig();
152
+ loadVoiceConfig();
chittios/static/style.css CHANGED
@@ -200,6 +200,41 @@ input[type="password"]:focus {
200
  color: var(--err);
201
  }
202
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
203
  .restart-note {
204
  margin-top: 1.4rem;
205
  padding-top: 1.1rem;
 
200
  color: var(--err);
201
  }
202
 
203
+ .voice {
204
+ margin-bottom: 1.4rem;
205
+ padding-top: 1.1rem;
206
+ border-top: 1px solid var(--line);
207
+ }
208
+
209
+ .voice .label {
210
+ margin-bottom: 0.6rem;
211
+ }
212
+
213
+ .voice-on {
214
+ display: flex;
215
+ align-items: baseline;
216
+ gap: 0.5rem;
217
+ flex-wrap: wrap;
218
+ font-size: 0.9rem;
219
+ color: var(--ok);
220
+ font-weight: 600;
221
+ }
222
+
223
+ .linkish {
224
+ font: inherit;
225
+ font-size: 0.85rem;
226
+ color: var(--muted);
227
+ background: none;
228
+ border: none;
229
+ padding: 0;
230
+ text-decoration: underline;
231
+ cursor: pointer;
232
+ }
233
+
234
+ .linkish:hover {
235
+ color: var(--accent);
236
+ }
237
+
238
  .restart-note {
239
  margin-top: 1.4rem;
240
  padding-top: 1.1rem;
chittios_core/expression/config.py ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Where the ElevenLabs voice credentials come from β€” env, then file, then off.
2
+
3
+ The local Piper voice needs no configuration and always works. This resolves the
4
+ *optional* upgrade: an ElevenLabs key (and which voice to speak in), so a
5
+ household that wants the premium cloud voice provides a key once and every later
6
+ start picks it up. No key anywhere means ``enabled`` is false and the robot keeps
7
+ its local voice β€” the safe default, chosen by absence.
8
+
9
+ Precedence: env var > config file > default, per field
10
+ ------------------------------------------------------
11
+ Mirrors the LLM config exactly, so the two behave the same to an operator:
12
+
13
+ 1. an environment variable (``ELEVENLABS_API_KEY`` for the key β€” the name the
14
+ ElevenLabs ecosystem already uses β€” and ``CHITTIOS_TTS_VOICE_ID`` /
15
+ ``CHITTIOS_TTS_MODEL`` for the rest);
16
+ 2. the on-disk config file (``~/.config/chittios/voice.json``) the settings UI
17
+ writes when a household pastes a key at onboarding;
18
+ 3. the built-in default β€” no key (so, off), a warm premade voice, the flash model.
19
+
20
+ The key is the one secret here, and it is treated like the LLM key: written to
21
+ the file, never echoed back over HTTP (the settings route reports only *whether*
22
+ one is set).
23
+
24
+ Why JSON, not TOML
25
+ ------------------
26
+ The settings UI writes this file, and the standard library reads and writes JSON
27
+ losslessly with no third-party dependency β€” the same reasoning, and the same
28
+ install floor, as the LLM config beside it.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import json
34
+ import os
35
+ from collections.abc import Mapping
36
+ from dataclasses import dataclass
37
+ from pathlib import Path
38
+ from typing import Final
39
+
40
+ from chittios_core.expression.elevenlabs import (
41
+ DEFAULT_ELEVENLABS_MODEL,
42
+ DEFAULT_ELEVENLABS_VOICE_ID,
43
+ )
44
+
45
+ #: The config file the settings UI writes and this module reads. Beside the LLM
46
+ #: config under ``~/.config/chittios`` by the XDG convention.
47
+ DEFAULT_CONFIG_PATH: Final = Path.home() / ".config" / "chittios" / "voice.json"
48
+
49
+ #: Environment overrides, highest precedence. The key uses the ElevenLabs-standard
50
+ #: name so an operator who already exports it needs no ChittiOS-specific lookup.
51
+ ENV_API_KEY: Final = "ELEVENLABS_API_KEY"
52
+ ENV_VOICE_ID: Final = "CHITTIOS_TTS_VOICE_ID"
53
+ ENV_MODEL: Final = "CHITTIOS_TTS_MODEL"
54
+
55
+ #: The JSON object keys, shared by reader and writer so the two never drift.
56
+ _KEY_API_KEY: Final = "api_key"
57
+ _KEY_VOICE_ID: Final = "voice_id"
58
+ _KEY_MODEL: Final = "model_id"
59
+
60
+
61
+ class VoiceConfigError(ValueError):
62
+ """Raised when the on-disk voice config exists but cannot be used.
63
+
64
+ A present-but-malformed file is an operator error worth surfacing rather than
65
+ papering over: silently ignoring it would drop a household's chosen voice
66
+ without a word. Construction of the engine is guarded upstream, so this
67
+ degrades TTS to the local voice rather than stopping the robot from booting.
68
+ """
69
+
70
+
71
+ @dataclass(frozen=True, slots=True)
72
+ class VoiceConfig:
73
+ """The resolved cloud-voice settings: an optional key, a voice, a model.
74
+
75
+ Frozen because it is a startup snapshot, read once and handed to the engine
76
+ factory. A change to the file or the environment takes effect on the next app
77
+ start, never by mutating a live config.
78
+
79
+ Attributes:
80
+ api_key: The ElevenLabs key, or ``None`` when none is configured β€” the
81
+ single fact that decides whether the cloud voice is used at all.
82
+ voice_id: The voice to speak in.
83
+ model_id: The synthesis model to request.
84
+ """
85
+
86
+ api_key: str | None
87
+ voice_id: str
88
+ model_id: str
89
+
90
+ @property
91
+ def enabled(self) -> bool:
92
+ """Whether the cloud voice should be used β€” i.e. a key is configured."""
93
+ return self.api_key is not None
94
+
95
+
96
+ def load_voice_config(
97
+ *,
98
+ path: Path = DEFAULT_CONFIG_PATH,
99
+ env: Mapping[str, str] | None = None,
100
+ ) -> VoiceConfig:
101
+ """Resolve the voice config: env var > file > default, per field.
102
+
103
+ Args:
104
+ path: The config file to read. A missing file is not an error β€” it means
105
+ "no key configured", and the cloud voice stays off.
106
+ env: The environment to read overrides from. Defaults to ``os.environ``;
107
+ injectable so a test can drive precedence without touching the
108
+ process environment.
109
+
110
+ Returns:
111
+ The resolved :class:`VoiceConfig`.
112
+
113
+ Raises:
114
+ VoiceConfigError: if the file exists but is not a JSON object, or a value
115
+ is present with the wrong type.
116
+ """
117
+ environ = os.environ if env is None else env
118
+ file_values = _read_config_file(path)
119
+
120
+ api_key = _resolve(ENV_API_KEY, _KEY_API_KEY, environ, file_values, None)
121
+ voice_id = (
122
+ _resolve(ENV_VOICE_ID, _KEY_VOICE_ID, environ, file_values, DEFAULT_ELEVENLABS_VOICE_ID)
123
+ or DEFAULT_ELEVENLABS_VOICE_ID
124
+ )
125
+ model_id = (
126
+ _resolve(ENV_MODEL, _KEY_MODEL, environ, file_values, DEFAULT_ELEVENLABS_MODEL)
127
+ or DEFAULT_ELEVENLABS_MODEL
128
+ )
129
+ return VoiceConfig(api_key=api_key, voice_id=voice_id, model_id=model_id)
130
+
131
+
132
+ def write_voice_config(config: VoiceConfig, *, path: Path = DEFAULT_CONFIG_PATH) -> None:
133
+ """Write ``config`` to ``path`` as JSON, creating the directory if needed.
134
+
135
+ The settings UI's writer: it persists what a household pastes at onboarding so
136
+ the next start resolves it. ``api_key`` is written as JSON ``null`` when
137
+ unset, keeping the file a faithful, complete snapshot of all three settings.
138
+
139
+ Args:
140
+ config: The settings to persist.
141
+ path: Where to write them. The parent directory is created if absent.
142
+ """
143
+ path.parent.mkdir(parents=True, exist_ok=True)
144
+ document = {
145
+ _KEY_API_KEY: config.api_key,
146
+ _KEY_VOICE_ID: config.voice_id,
147
+ _KEY_MODEL: config.model_id,
148
+ }
149
+ path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
150
+
151
+
152
+ def _read_config_file(path: Path) -> Mapping[str, object]:
153
+ """Read the config file into a mapping; an empty mapping if it does not exist.
154
+
155
+ Raises:
156
+ VoiceConfigError: if the file exists but does not hold a JSON object.
157
+ """
158
+ if not path.exists():
159
+ return {}
160
+ try:
161
+ parsed: object = json.loads(path.read_text(encoding="utf-8"))
162
+ except (OSError, json.JSONDecodeError) as exc:
163
+ raise VoiceConfigError(f"could not read the voice config at {path}: {exc}") from exc
164
+ if not isinstance(parsed, dict):
165
+ raise VoiceConfigError(
166
+ f"the voice config at {path} must be a JSON object, got {type(parsed).__name__}"
167
+ )
168
+ return parsed
169
+
170
+
171
+ def _resolve(
172
+ env_key: str,
173
+ file_key: str,
174
+ env: Mapping[str, str],
175
+ file_values: Mapping[str, object],
176
+ default: str | None,
177
+ ) -> str | None:
178
+ """Return one field's value: env var > file entry > default.
179
+
180
+ A blank or whitespace-only environment variable counts as unset, so an
181
+ exported-but-empty ``ELEVENLABS_API_KEY`` falls through to the file rather
182
+ than pinning the key to the empty string. A file entry present as JSON
183
+ ``null`` likewise means "unset" and falls through to the default.
184
+ """
185
+ env_value = env.get(env_key)
186
+ if env_value is not None and env_value.strip():
187
+ return env_value
188
+
189
+ if file_key in file_values:
190
+ file_value = file_values[file_key]
191
+ if file_value is None:
192
+ return default
193
+ if not isinstance(file_value, str):
194
+ raise VoiceConfigError(
195
+ f"the voice config value for {file_key!r} must be a string or null, "
196
+ f"got {type(file_value).__name__}"
197
+ )
198
+ return file_value
199
+
200
+ return default
chittios_core/expression/elevenlabs.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ElevenLabs cloud text-to-speech β€” an optional, premium voice (CHT-TTS-cloud).
2
+
3
+ The local Piper voice is the default and the privacy floor: it never leaves the
4
+ box. This is the opposite trade a household can *opt into* β€” the robot's reply
5
+ text is sent to ElevenLabs and natural, expressive speech comes back β€” chosen by
6
+ pasting an API key at onboarding. It is deliberately not the default: a key must
7
+ be present for this engine to be built at all, and when it is absent, or fails,
8
+ the local voice speaks instead (see :class:`~chittios_core.expression.tts.FallbackTtsEngine`).
9
+
10
+ What crosses the wire, and what does not
11
+ ----------------------------------------
12
+ Only the *reply* β€” text ChittiOS generated β€” goes to ElevenLabs. The family's
13
+ microphone audio never does; speech-to-text stays local by construction (that is
14
+ the line the project does not cross). Sending the robot's own words out for
15
+ synthesis is the same shape of trade the household already made when they pointed
16
+ conversation at a chosen LLM endpoint.
17
+
18
+ PCM straight onto the audio spine
19
+ ---------------------------------
20
+ The request asks for ``pcm_16000`` β€” signed 16-bit little-endian mono at 16 kHz,
21
+ which *is* the rate the whole audio path standardises on. So the response decodes
22
+ with a single ``frombuffer`` and an int16β†’float32 scale: no MP3 decoder
23
+ dependency, and ``speak`` resamples nothing because the rate already matches.
24
+
25
+ Failures name themselves, and never mute the robot
26
+ --------------------------------------------------
27
+ A 4xx β€” a bad or unauthorised key, a plan that does not permit PCM, an exhausted
28
+ quota β€” is a standing condition, raised as :class:`TtsUnavailableError` so the
29
+ fallback engine latches to the local voice for the session. A 5xx or a network
30
+ blip is transient, raised as :class:`TtsTransientError` so the fallback covers
31
+ just that one line and the next line tries the cloud again.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ from typing import Final
37
+
38
+ import httpx
39
+ import numpy as np
40
+ from numpy.typing import NDArray
41
+
42
+ from chittios_core.expression.tts import TtsTransientError, TtsUnavailableError
43
+ from hal.reference_hal import AUDIO_DTYPE
44
+
45
+ #: ElevenLabs' low-latency model β€” fast and inexpensive per character, the right
46
+ #: default for a robot that speaks in short conversational turns rather than
47
+ #: narrating essays. Overridable per household via config.
48
+ DEFAULT_ELEVENLABS_MODEL: Final = "eleven_flash_v2_5"
49
+
50
+ #: "George" β€” a warm, premade storyteller voice. Premade voice ids are shared
51
+ #: across every ElevenLabs account, so this default resolves for any customer's
52
+ #: key, not just the developer's. Overridable per household via config.
53
+ DEFAULT_ELEVENLABS_VOICE_ID: Final = "JBFqnCBsd6RMkjVDRZzb"
54
+
55
+ #: The API host. A constant, not a config field: unlike the LLM endpoint there is
56
+ #: one ElevenLabs, and a household chooses the voice, not the server.
57
+ DEFAULT_ELEVENLABS_BASE_URL: Final = "https://api.elevenlabs.io"
58
+
59
+ #: The one output format requested, and the rate it carries. Both are fixed
60
+ #: together: ``pcm_16000`` is signed 16-bit LE mono at exactly this rate, so the
61
+ #: decode is a ``frombuffer`` and no resample is ever needed downstream.
62
+ _OUTPUT_FORMAT: Final = "pcm_16000"
63
+ _SAMPLE_RATE_HZ: Final = 16_000
64
+
65
+ #: How long to wait on one synthesis before calling it a transient failure. Long
66
+ #: enough for a sentence on a slow link, short enough that a hung request falls
67
+ #: back to the local voice rather than stalling the turn.
68
+ _DEFAULT_TIMEOUT_S: Final = 30.0
69
+
70
+
71
+ class ElevenLabsTtsEngine:
72
+ """Synthesizes text through the ElevenLabs cloud API, returning 16 kHz mono.
73
+
74
+ Satisfies the :class:`~chittios_core.expression.tts.TtsEngine` protocol, so it
75
+ drops into ``speak`` exactly where Piper or the stub would. Build it only when
76
+ a key is configured; wrap it in a ``FallbackTtsEngine`` so a cloud failure is
77
+ covered by the local voice rather than heard as silence.
78
+
79
+ Args:
80
+ api_key: The household's ElevenLabs API key. Required and non-empty.
81
+ voice_id: The voice to speak in. Defaults to a warm premade voice.
82
+ model_id: The synthesis model. Defaults to the low-latency flash model.
83
+ base_url: The API host. Defaults to ElevenLabs.
84
+ timeout_s: Per-request timeout in seconds.
85
+ client: An injected HTTP client, for tests. Defaults to a pooled
86
+ :class:`httpx.Client`, so successive turns reuse one TLS connection
87
+ rather than paying a fresh handshake each time.
88
+
89
+ Raises:
90
+ TtsUnavailableError: if ``api_key`` is empty β€” the one failure knowable at
91
+ construction, kept here so a blank key never yields a live-but-useless
92
+ engine.
93
+ """
94
+
95
+ def __init__(
96
+ self,
97
+ api_key: str,
98
+ *,
99
+ voice_id: str = DEFAULT_ELEVENLABS_VOICE_ID,
100
+ model_id: str = DEFAULT_ELEVENLABS_MODEL,
101
+ base_url: str = DEFAULT_ELEVENLABS_BASE_URL,
102
+ timeout_s: float = _DEFAULT_TIMEOUT_S,
103
+ client: httpx.Client | None = None,
104
+ ) -> None:
105
+ if not api_key or not api_key.strip():
106
+ raise TtsUnavailableError("ElevenLabsTtsEngine needs a non-empty API key.")
107
+ self._api_key = api_key
108
+ self._model_id = model_id
109
+ self._url = f"{base_url.rstrip('/')}/v1/text-to-speech/{voice_id}"
110
+ self._client = client if client is not None else httpx.Client(timeout=timeout_s)
111
+
112
+ def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
113
+ """Synthesize ``text`` and return ``(mono float32 samples, 16000)``.
114
+
115
+ Whitespace-only or empty text short-circuits to empty audio β€” there is
116
+ nothing to say and no reason to spend a request or a character of quota.
117
+
118
+ Raises:
119
+ TtsUnavailableError: on a 4xx β€” a standing condition (bad key, a plan
120
+ without PCM, exhausted quota) the caller should stop retrying.
121
+ TtsTransientError: on a 5xx or a network error β€” worth trying again on
122
+ the next line.
123
+ """
124
+ if not text.strip():
125
+ return np.zeros(0, dtype=AUDIO_DTYPE), _SAMPLE_RATE_HZ
126
+
127
+ try:
128
+ response = self._client.post(
129
+ self._url,
130
+ params={"output_format": _OUTPUT_FORMAT},
131
+ headers={"xi-api-key": self._api_key, "content-type": "application/json"},
132
+ json={"text": text, "model_id": self._model_id},
133
+ )
134
+ except httpx.HTTPError as exc:
135
+ raise TtsTransientError(f"could not reach ElevenLabs: {exc}") from exc
136
+
137
+ if response.status_code != 200:
138
+ detail = _short_detail(response)
139
+ if 400 <= response.status_code < 500:
140
+ raise TtsUnavailableError(
141
+ f"ElevenLabs rejected the request (HTTP {response.status_code}): {detail}"
142
+ )
143
+ raise TtsTransientError(
144
+ f"ElevenLabs is unavailable (HTTP {response.status_code}): {detail}"
145
+ )
146
+
147
+ pcm = np.frombuffer(response.content, dtype="<i2")
148
+ # int16 β†’ float32 in [-1, 1). Dividing by 32768 (not 32767) keeps a
149
+ # full-scale negative sample exactly at -1.0, matching the convention the
150
+ # rest of the audio spine uses.
151
+ samples = (pcm.astype(np.float32) / 32768.0).astype(AUDIO_DTYPE)
152
+ return samples, _SAMPLE_RATE_HZ
153
+
154
+
155
+ def _short_detail(response: httpx.Response) -> str:
156
+ """A brief, safe description of a failed response for a log line.
157
+
158
+ The body is truncated because ElevenLabs' error JSON can be long, and this
159
+ lands in a warning the household may read β€” the status and a hint, not a wall.
160
+ """
161
+ try:
162
+ return response.text[:200]
163
+ except Exception: # pragma: no cover - a body that will not decode is still a failure
164
+ return "<unreadable response body>"
chittios_core/expression/tts.py CHANGED
@@ -1,305 +1,373 @@
1
- """Local text-to-speech for the expression subsystem (CHT-019, Phase 2).
2
-
3
- Layer 2's voice. Core turns a line of text into audio and streams it to the
4
- body through the HAL contract's ``audio_out``. Everything here runs on the
5
- machine β€” no cloud, no network β€” which is the whole promise of the project:
6
- *"With the local setup, audio never leaves your machine."* (spec Β§9).
7
-
8
- Three deliberate shapes
9
- -----------------------
10
- **A ``TtsEngine`` seam.** Synthesis is the one part that wants a heavy,
11
- optional runtime (an ONNX voice model). Putting it behind a Protocol keeps the
12
- streaming, resampling, and chunking logic β€” the part Core actually depends on β€”
13
- testable without that runtime, the same way ``identity.voice`` keeps its
14
- matching policy testable without torch. A ``StubTtsEngine`` satisfies the seam
15
- with deterministic synthetic audio; ``PiperTtsEngine`` is the real backend and
16
- is imported lazily so this module stays importable on a machine with no Piper.
17
-
18
- **Everything lands at 16 kHz float32 mono.** The Reachy Mini push path
19
- (``media.push_audio_sample``) wants float32 at 16 kHz, and the HAL contract
20
- fixes float32 as the processing format end to end (``AUDIO_DTYPE``). A voice
21
- model that speaks at 22.05 kHz is normal, so ``speak`` resamples whatever the
22
- engine produces onto the target rate rather than trusting engines to agree.
23
- Resampling once, here, means no downstream stage has to.
24
-
25
- **Chunked, not buffered.** ``speak`` emits ~100 ms ``AudioChunk``s as it goes
26
- so the body can begin speaking before the whole utterance is synthesized and
27
- resampled. That streaming is what the narration-latency budget in spec Β§7.2
28
- (time-to-first-audio) is built on; buffering the full clip first would spend
29
- the entire budget before the first sample reached the speaker.
30
- """
31
-
32
- from __future__ import annotations
33
-
34
- from collections.abc import AsyncIterator
35
- from math import gcd
36
- from typing import Final, Protocol, runtime_checkable
37
-
38
- import numpy as np
39
- from numpy.typing import NDArray
40
-
41
- from hal.reference_hal import AUDIO_DTYPE, AudioChunk
42
-
43
- #: The rate every utterance is delivered at. The Reachy Mini push path expects
44
- #: float32 @ 16 kHz, and 16 kHz is already the rate the perception and voice
45
- #: paths standardize on, so one constant governs the whole audio spine.
46
- TARGET_SAMPLE_RATE_HZ: Final = 16_000
47
-
48
- #: Chunk granularity for streaming playback. ~100 ms is small enough that
49
- #: time-to-first-audio stays inside the budget yet large enough that the
50
- #: per-chunk overhead (an ``AudioChunk`` allocation and an ``await``) is noise.
51
- DEFAULT_CHUNK_MS: Final = 100
52
-
53
- #: Piper's default voices speak at 22.05 kHz. Used by ``StubTtsEngine`` so the
54
- #: default test path exercises the resampler rather than the pass-through.
55
- DEFAULT_STUB_SAMPLE_RATE_HZ: Final = 22_050
56
-
57
-
58
- class TtsUnavailableError(RuntimeError):
59
- """Raised when a TTS backend's runtime dependency is not installed.
60
-
61
- Constructing a backend is where the missing dependency surfaces, so the
62
- failure names the extra to install rather than letting an ``ImportError``
63
- escape from deep inside synthesis with no remedy attached.
64
- """
65
-
66
-
67
- @runtime_checkable
68
- class TtsEngine(Protocol):
69
- """Turns text into a single block of mono audio at the engine's own rate.
70
-
71
- The engine owns synthesis and nothing else: it does not resample, chunk, or
72
- touch the body. Returning the rate alongside the samples β€” rather than
73
- assuming a fixed one β€” is what lets ``speak`` accept any voice model and
74
- still guarantee 16 kHz downstream.
75
- """
76
-
77
- def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
78
- """Return ``(mono float32 samples in [-1, 1], sample_rate_hz)``."""
79
- ...
80
-
81
-
82
- class StubTtsEngine:
83
- """Deterministic synthetic speech for tests β€” no external dependency.
84
-
85
- Emits a fixed-frequency tone whose length grows with the text, so callers
86
- get non-empty, chunk-able audio without a voice model. Deterministic by
87
- construction: the same text always yields identical samples, which is what
88
- lets tests assert on exact behaviour instead of tolerating model jitter.
89
-
90
- Args:
91
- sample_rate_hz: Rate of the produced audio. Defaults to Piper's
92
- 22.05 kHz so the default path deliberately differs from the 16 kHz
93
- target and exercises the resampler.
94
- tone_hz: Frequency of the synthesized tone.
95
- seconds_per_char: Audio duration contributed by each character.
96
- amplitude: Peak amplitude, comfortably inside [-1, 1].
97
- """
98
-
99
- def __init__(
100
- self,
101
- *,
102
- sample_rate_hz: int = DEFAULT_STUB_SAMPLE_RATE_HZ,
103
- tone_hz: float = 220.0,
104
- seconds_per_char: float = 0.08,
105
- amplitude: float = 0.25,
106
- ) -> None:
107
- if sample_rate_hz <= 0:
108
- raise ValueError(f"sample_rate_hz must be positive, got {sample_rate_hz}")
109
- if tone_hz <= 0:
110
- raise ValueError(f"tone_hz must be positive, got {tone_hz}")
111
- if seconds_per_char <= 0:
112
- raise ValueError(f"seconds_per_char must be positive, got {seconds_per_char}")
113
- if not 0.0 < amplitude <= 1.0:
114
- raise ValueError(f"amplitude must lie in (0, 1], got {amplitude}")
115
- self._sample_rate_hz = sample_rate_hz
116
- self._tone_hz = tone_hz
117
- self._seconds_per_char = seconds_per_char
118
- self._amplitude = amplitude
119
-
120
- def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
121
- """Return a tone whose duration scales with ``len(text)``.
122
-
123
- Whitespace-only or empty text yields an empty array β€” the honest
124
- representation of "nothing to say" β€” which ``speak`` streams as zero
125
- chunks rather than a silent blip.
126
- """
127
- char_count = len(text.strip())
128
- if char_count == 0:
129
- return np.zeros(0, dtype=AUDIO_DTYPE), self._sample_rate_hz
130
-
131
- duration_s = char_count * self._seconds_per_char
132
- sample_count = int(self._sample_rate_hz * duration_s)
133
- t = np.arange(sample_count, dtype=np.float64) / self._sample_rate_hz
134
- samples = self._amplitude * np.sin(2.0 * np.pi * self._tone_hz * t)
135
- return samples.astype(AUDIO_DTYPE), self._sample_rate_hz
136
-
137
-
138
- class PiperTtsEngine:
139
- """Piper-backed synthesis, behind the optional ``voice-tts`` extra.
140
-
141
- Piper is a local, ONNX-based neural TTS: no cloud, fast on CPU, which is
142
- exactly the constraint a household robot's voice lives under. It is an
143
- optional dependency β€” importing this module never requires it β€” so the
144
- import is deferred to construction and a clear ``TtsUnavailableError`` is
145
- raised when it is absent, naming the extra to install.
146
-
147
- Args:
148
- model_path: Path to a Piper ``.onnx`` voice model.
149
- config_path: Path to the model's ``.onnx.json`` config. Piper defaults
150
- to ``<model_path>.json`` when omitted, so this is only needed when
151
- the config sits elsewhere.
152
- use_cuda: Run inference on GPU. Off by default β€” the target is a CPU
153
- SBC, and CUDA is the exception, not the rule.
154
-
155
- Raises:
156
- TtsUnavailableError: if ``piper-tts`` is not installed.
157
- """
158
-
159
- def __init__(
160
- self,
161
- model_path: str,
162
- *,
163
- config_path: str | None = None,
164
- use_cuda: bool = False,
165
- ) -> None:
166
- try:
167
- from piper import PiperVoice
168
- except ImportError as exc: # pragma: no cover - exercised only without piper
169
- raise TtsUnavailableError(
170
- "PiperTtsEngine needs the 'piper-tts' package, which is not installed. "
171
- "Install the optional extra: pip install 'chittios[voice-tts]'."
172
- ) from exc
173
-
174
- self._voice = PiperVoice.load(model_path, config_path=config_path, use_cuda=use_cuda)
175
-
176
- def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
177
- """Synthesize ``text`` and return ``(mono float32 samples, rate)``.
178
-
179
- Piper streams int16 chunks; they are concatenated and converted to the
180
- float32 the rest of the pipeline speaks in. ``speak`` re-chunks the
181
- result for playback, so gathering Piper's chunks here keeps this method
182
- a plain ``TtsEngine`` β€” text in, one waveform out β€” and leaves streaming
183
- policy in one place.
184
-
185
- Raises:
186
- TtsUnavailableError: if Piper yields no audio for the text.
187
- """
188
- blocks: list[NDArray[np.int16]] = []
189
- sample_rate: int | None = None
190
- for chunk in self._voice.synthesize(text):
191
- sample_rate = chunk.sample_rate
192
- blocks.append(np.frombuffer(chunk.audio_int16_bytes, dtype=np.int16))
193
-
194
- if sample_rate is None:
195
- raise TtsUnavailableError(f"Piper produced no audio for text: {text!r}")
196
-
197
- if blocks:
198
- pcm = np.concatenate(blocks)
199
- else:
200
- pcm = np.zeros(0, dtype=np.int16)
201
-
202
- # int16 -> float32 in [-1, 1). Dividing by 32768 (not 32767) is the
203
- # standard convention: it keeps full-scale negative samples exactly at
204
- # -1.0 and never overshoots the range the contract promises.
205
- samples = (pcm.astype(np.float32) / 32768.0).astype(AUDIO_DTYPE)
206
- return samples, sample_rate
207
-
208
-
209
- def _resample_to_target(samples: NDArray[np.float32], source_rate_hz: int) -> NDArray[np.float32]:
210
- """Resample mono audio onto ``TARGET_SAMPLE_RATE_HZ`` float32.
211
-
212
- A pass-through when the rates already match, so the common Piper-at-16 kHz
213
- case pays nothing. Otherwise polyphase resampling via ``scipy.signal`` β€”
214
- imported lazily, matching the HAL adapters, so this module stays importable
215
- without SciPy installed. Reducing up/down by their GCD keeps the polyphase
216
- filter small (16 kHz from 22.05 kHz is 320/441, not 16000/22050).
217
-
218
- Args:
219
- samples: Mono float32 at ``source_rate_hz``.
220
- source_rate_hz: The rate ``samples`` were produced at.
221
-
222
- Returns:
223
- Contiguous float32 at ``TARGET_SAMPLE_RATE_HZ``, clipped to [-1, 1]
224
- because polyphase filtering can ring a hair past full scale.
225
- """
226
- if samples.ndim != 1:
227
- raise ValueError(f"expected mono (1-D) audio, got shape {samples.shape}")
228
- if source_rate_hz <= 0:
229
- raise ValueError(f"source_rate_hz must be positive, got {source_rate_hz}")
230
- if source_rate_hz == TARGET_SAMPLE_RATE_HZ or samples.size == 0:
231
- return np.ascontiguousarray(samples, dtype=AUDIO_DTYPE)
232
-
233
- from scipy.signal import resample_poly
234
-
235
- divisor = gcd(source_rate_hz, TARGET_SAMPLE_RATE_HZ)
236
- up = TARGET_SAMPLE_RATE_HZ // divisor
237
- down = source_rate_hz // divisor
238
- resampled = resample_poly(samples.astype(np.float64), up, down)
239
- return np.ascontiguousarray(np.clip(resampled, -1.0, 1.0), dtype=AUDIO_DTYPE)
240
-
241
-
242
- async def _stream_chunks(
243
- samples: NDArray[np.float32], *, chunk_ms: int
244
- ) -> AsyncIterator[AudioChunk]:
245
- """Slice 16 kHz mono ``samples`` into ~``chunk_ms`` ``AudioChunk``s.
246
-
247
- Timestamps advance by each chunk's own duration so the stream carries a
248
- coherent playback clock. A short final chunk is yielded as-is rather than
249
- padded β€” padding would inject silence the speaker would actually play.
250
- """
251
- samples_per_chunk = int(TARGET_SAMPLE_RATE_HZ * chunk_ms / 1000)
252
- elapsed_s = 0.0
253
- for start in range(0, len(samples), samples_per_chunk):
254
- block = np.ascontiguousarray(samples[start : start + samples_per_chunk], dtype=AUDIO_DTYPE)
255
- yield AudioChunk(
256
- samples=block,
257
- sample_rate_hz=TARGET_SAMPLE_RATE_HZ,
258
- channels=1,
259
- timestamp_s=elapsed_s,
260
- )
261
- elapsed_s += len(block) / TARGET_SAMPLE_RATE_HZ
262
-
263
-
264
- @runtime_checkable
265
- class SupportsAudioOut(Protocol):
266
- """The slice of a HAL body that ``speak`` needs: a sink for an audio stream.
267
-
268
- Typing ``speak`` to this narrow shape (not the full ``HALAdapter``) is what
269
- lets the screen-avatar test double β€” which implements only ``audio_out`` β€”
270
- stand in for the real Reachy body, and keeps ``speak`` honest about what it
271
- actually touches.
272
- """
273
-
274
- async def audio_out(self, chunks: AsyncIterator[AudioChunk]) -> None:
275
- """Play a stream of audio chunks, returning once they are consumed."""
276
- ...
277
-
278
-
279
- async def speak(
280
- text: str,
281
- body: SupportsAudioOut,
282
- engine: TtsEngine,
283
- *,
284
- chunk_ms: int = DEFAULT_CHUNK_MS,
285
- ) -> None:
286
- """Synthesize ``text`` and stream it to ``body`` as 16 kHz float32 mono.
287
-
288
- The full expression path for one line: synthesize with ``engine``, resample
289
- to the body's expected rate, and stream ~``chunk_ms`` chunks into
290
- ``body.audio_out`` so playback can start before synthesis finishes. Empty
291
- or whitespace-only text streams nothing β€” there is no audio to play, and an
292
- empty stream is the correct representation of silence.
293
-
294
- Args:
295
- text: The line to speak.
296
- body: The HAL adapter that will play the audio.
297
- engine: The synthesis backend.
298
- chunk_ms: Playback chunk duration. Must be positive.
299
- """
300
- if chunk_ms <= 0:
301
- raise ValueError(f"chunk_ms must be positive, got {chunk_ms}")
302
-
303
- raw_samples, source_rate_hz = engine.synthesize(text)
304
- samples = _resample_to_target(raw_samples, source_rate_hz)
305
- await body.audio_out(_stream_chunks(samples, chunk_ms=chunk_ms))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Local text-to-speech for the expression subsystem (CHT-019, Phase 2).
2
+
3
+ Layer 2's voice. Core turns a line of text into audio and streams it to the
4
+ body through the HAL contract's ``audio_out``. Everything here runs on the
5
+ machine β€” no cloud, no network β€” which is the whole promise of the project:
6
+ *"With the local setup, audio never leaves your machine."* (spec Β§9).
7
+
8
+ Three deliberate shapes
9
+ -----------------------
10
+ **A ``TtsEngine`` seam.** Synthesis is the one part that wants a heavy,
11
+ optional runtime (an ONNX voice model). Putting it behind a Protocol keeps the
12
+ streaming, resampling, and chunking logic β€” the part Core actually depends on β€”
13
+ testable without that runtime, the same way ``identity.voice`` keeps its
14
+ matching policy testable without torch. A ``StubTtsEngine`` satisfies the seam
15
+ with deterministic synthetic audio; ``PiperTtsEngine`` is the real backend and
16
+ is imported lazily so this module stays importable on a machine with no Piper.
17
+
18
+ **Everything lands at 16 kHz float32 mono.** The Reachy Mini push path
19
+ (``media.push_audio_sample``) wants float32 at 16 kHz, and the HAL contract
20
+ fixes float32 as the processing format end to end (``AUDIO_DTYPE``). A voice
21
+ model that speaks at 22.05 kHz is normal, so ``speak`` resamples whatever the
22
+ engine produces onto the target rate rather than trusting engines to agree.
23
+ Resampling once, here, means no downstream stage has to.
24
+
25
+ **Chunked, not buffered.** ``speak`` emits ~100 ms ``AudioChunk``s as it goes
26
+ so the body can begin speaking before the whole utterance is synthesized and
27
+ resampled. That streaming is what the narration-latency budget in spec Β§7.2
28
+ (time-to-first-audio) is built on; buffering the full clip first would spend
29
+ the entire budget before the first sample reached the speaker.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import asyncio
35
+ import logging
36
+ from collections.abc import AsyncIterator
37
+ from math import gcd
38
+ from typing import Final, Protocol, runtime_checkable
39
+
40
+ import numpy as np
41
+ from numpy.typing import NDArray
42
+
43
+ from hal.reference_hal import AUDIO_DTYPE, AudioChunk
44
+
45
+ logger = logging.getLogger(__name__)
46
+
47
+ #: The rate every utterance is delivered at. The Reachy Mini push path expects
48
+ #: float32 @ 16 kHz, and 16 kHz is already the rate the perception and voice
49
+ #: paths standardize on, so one constant governs the whole audio spine.
50
+ TARGET_SAMPLE_RATE_HZ: Final = 16_000
51
+
52
+ #: Chunk granularity for streaming playback. ~100 ms is small enough that
53
+ #: time-to-first-audio stays inside the budget yet large enough that the
54
+ #: per-chunk overhead (an ``AudioChunk`` allocation and an ``await``) is noise.
55
+ DEFAULT_CHUNK_MS: Final = 100
56
+
57
+ #: Piper's default voices speak at 22.05 kHz. Used by ``StubTtsEngine`` so the
58
+ #: default test path exercises the resampler rather than the pass-through.
59
+ DEFAULT_STUB_SAMPLE_RATE_HZ: Final = 22_050
60
+
61
+
62
+ class TtsUnavailableError(RuntimeError):
63
+ """Raised when a TTS backend cannot be used, and the caller should stop trying.
64
+
65
+ Two shapes of standing failure: a backend's runtime dependency is not
66
+ installed (surfaced at construction, naming the extra to install), or a cloud
67
+ backend was refused for a durable reason β€” a bad key, a plan without the
68
+ requested format, an exhausted quota (a 4xx). Both mean "do not keep retrying
69
+ this backend"; :class:`FallbackTtsEngine` latches to its fallback on this.
70
+ """
71
+
72
+
73
+ class TtsTransientError(RuntimeError):
74
+ """Raised when a TTS backend failed in a way that may succeed next time.
75
+
76
+ A network blip or a 5xx from a cloud backend β€” worth covering with the
77
+ fallback for just this one line, then trying the primary again on the next.
78
+ The counterpart to :class:`TtsUnavailableError`, which is durable.
79
+ """
80
+
81
+
82
+ @runtime_checkable
83
+ class TtsEngine(Protocol):
84
+ """Turns text into a single block of mono audio at the engine's own rate.
85
+
86
+ The engine owns synthesis and nothing else: it does not resample, chunk, or
87
+ touch the body. Returning the rate alongside the samples β€” rather than
88
+ assuming a fixed one β€” is what lets ``speak`` accept any voice model and
89
+ still guarantee 16 kHz downstream.
90
+ """
91
+
92
+ def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
93
+ """Return ``(mono float32 samples in [-1, 1], sample_rate_hz)``."""
94
+ ...
95
+
96
+
97
+ class StubTtsEngine:
98
+ """Deterministic synthetic speech for tests β€” no external dependency.
99
+
100
+ Emits a fixed-frequency tone whose length grows with the text, so callers
101
+ get non-empty, chunk-able audio without a voice model. Deterministic by
102
+ construction: the same text always yields identical samples, which is what
103
+ lets tests assert on exact behaviour instead of tolerating model jitter.
104
+
105
+ Args:
106
+ sample_rate_hz: Rate of the produced audio. Defaults to Piper's
107
+ 22.05 kHz so the default path deliberately differs from the 16 kHz
108
+ target and exercises the resampler.
109
+ tone_hz: Frequency of the synthesized tone.
110
+ seconds_per_char: Audio duration contributed by each character.
111
+ amplitude: Peak amplitude, comfortably inside [-1, 1].
112
+ """
113
+
114
+ def __init__(
115
+ self,
116
+ *,
117
+ sample_rate_hz: int = DEFAULT_STUB_SAMPLE_RATE_HZ,
118
+ tone_hz: float = 220.0,
119
+ seconds_per_char: float = 0.08,
120
+ amplitude: float = 0.25,
121
+ ) -> None:
122
+ if sample_rate_hz <= 0:
123
+ raise ValueError(f"sample_rate_hz must be positive, got {sample_rate_hz}")
124
+ if tone_hz <= 0:
125
+ raise ValueError(f"tone_hz must be positive, got {tone_hz}")
126
+ if seconds_per_char <= 0:
127
+ raise ValueError(f"seconds_per_char must be positive, got {seconds_per_char}")
128
+ if not 0.0 < amplitude <= 1.0:
129
+ raise ValueError(f"amplitude must lie in (0, 1], got {amplitude}")
130
+ self._sample_rate_hz = sample_rate_hz
131
+ self._tone_hz = tone_hz
132
+ self._seconds_per_char = seconds_per_char
133
+ self._amplitude = amplitude
134
+
135
+ def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
136
+ """Return a tone whose duration scales with ``len(text)``.
137
+
138
+ Whitespace-only or empty text yields an empty array β€” the honest
139
+ representation of "nothing to say" β€” which ``speak`` streams as zero
140
+ chunks rather than a silent blip.
141
+ """
142
+ char_count = len(text.strip())
143
+ if char_count == 0:
144
+ return np.zeros(0, dtype=AUDIO_DTYPE), self._sample_rate_hz
145
+
146
+ duration_s = char_count * self._seconds_per_char
147
+ sample_count = int(self._sample_rate_hz * duration_s)
148
+ t = np.arange(sample_count, dtype=np.float64) / self._sample_rate_hz
149
+ samples = self._amplitude * np.sin(2.0 * np.pi * self._tone_hz * t)
150
+ return samples.astype(AUDIO_DTYPE), self._sample_rate_hz
151
+
152
+
153
+ class PiperTtsEngine:
154
+ """Piper-backed synthesis, behind the optional ``voice-tts`` extra.
155
+
156
+ Piper is a local, ONNX-based neural TTS: no cloud, fast on CPU, which is
157
+ exactly the constraint a household robot's voice lives under. It is an
158
+ optional dependency β€” importing this module never requires it β€” so the
159
+ import is deferred to construction and a clear ``TtsUnavailableError`` is
160
+ raised when it is absent, naming the extra to install.
161
+
162
+ Args:
163
+ model_path: Path to a Piper ``.onnx`` voice model.
164
+ config_path: Path to the model's ``.onnx.json`` config. Piper defaults
165
+ to ``<model_path>.json`` when omitted, so this is only needed when
166
+ the config sits elsewhere.
167
+ use_cuda: Run inference on GPU. Off by default β€” the target is a CPU
168
+ SBC, and CUDA is the exception, not the rule.
169
+
170
+ Raises:
171
+ TtsUnavailableError: if ``piper-tts`` is not installed.
172
+ """
173
+
174
+ def __init__(
175
+ self,
176
+ model_path: str,
177
+ *,
178
+ config_path: str | None = None,
179
+ use_cuda: bool = False,
180
+ ) -> None:
181
+ try:
182
+ from piper import PiperVoice
183
+ except ImportError as exc: # pragma: no cover - exercised only without piper
184
+ raise TtsUnavailableError(
185
+ "PiperTtsEngine needs the 'piper-tts' package, which is not installed. "
186
+ "Install the optional extra: pip install 'chittios[voice-tts]'."
187
+ ) from exc
188
+
189
+ self._voice = PiperVoice.load(model_path, config_path=config_path, use_cuda=use_cuda)
190
+
191
+ def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
192
+ """Synthesize ``text`` and return ``(mono float32 samples, rate)``.
193
+
194
+ Piper streams int16 chunks; they are concatenated and converted to the
195
+ float32 the rest of the pipeline speaks in. ``speak`` re-chunks the
196
+ result for playback, so gathering Piper's chunks here keeps this method
197
+ a plain ``TtsEngine`` β€” text in, one waveform out β€” and leaves streaming
198
+ policy in one place.
199
+
200
+ Raises:
201
+ TtsUnavailableError: if Piper yields no audio for the text.
202
+ """
203
+ blocks: list[NDArray[np.int16]] = []
204
+ sample_rate: int | None = None
205
+ for chunk in self._voice.synthesize(text):
206
+ sample_rate = chunk.sample_rate
207
+ blocks.append(np.frombuffer(chunk.audio_int16_bytes, dtype=np.int16))
208
+
209
+ if sample_rate is None:
210
+ raise TtsUnavailableError(f"Piper produced no audio for text: {text!r}")
211
+
212
+ if blocks:
213
+ pcm = np.concatenate(blocks)
214
+ else:
215
+ pcm = np.zeros(0, dtype=np.int16)
216
+
217
+ # int16 -> float32 in [-1, 1). Dividing by 32768 (not 32767) is the
218
+ # standard convention: it keeps full-scale negative samples exactly at
219
+ # -1.0 and never overshoots the range the contract promises.
220
+ samples = (pcm.astype(np.float32) / 32768.0).astype(AUDIO_DTYPE)
221
+ return samples, sample_rate
222
+
223
+
224
+ class FallbackTtsEngine:
225
+ """Speaks through a primary engine, falling back to a local one on failure.
226
+
227
+ The one place the cloud/local trade is arbitrated: a household that has opted
228
+ into ElevenLabs gets it, but a bad key, an exhausted quota, or a dropped
229
+ connection is never heard as silence β€” the local voice covers the line
230
+ instead. This keeps every other layer unaware there is a choice at all; it
231
+ still holds one ``TtsEngine`` and calls ``synthesize``.
232
+
233
+ Durable vs transient, from the raised error:
234
+
235
+ - :class:`TtsUnavailableError` is a standing condition (bad key, plan without
236
+ PCM, quota gone). The primary is *disabled for the session* β€” retrying it
237
+ every sentence would add a doomed round-trip to each turn β€” and a restart is
238
+ what re-checks it, once the household has fixed the cause.
239
+ - :class:`TtsTransientError` (or any unexpected error) covers just this line;
240
+ the next line tries the primary again.
241
+
242
+ Args:
243
+ primary: The preferred engine (e.g. ElevenLabs).
244
+ fallback: The always-available local engine (Piper, or the stub tone).
245
+ """
246
+
247
+ def __init__(self, primary: TtsEngine, fallback: TtsEngine) -> None:
248
+ self._primary = primary
249
+ self._fallback = fallback
250
+ self._primary_disabled = False
251
+
252
+ def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
253
+ """Synthesize with the primary engine, or the fallback if it cannot."""
254
+ if not self._primary_disabled:
255
+ try:
256
+ return self._primary.synthesize(text)
257
+ except TtsUnavailableError as exc:
258
+ logger.warning(
259
+ "primary voice unavailable (%s); using the local voice for the rest "
260
+ "of this session β€” restart once the cause is fixed to re-enable it",
261
+ exc,
262
+ )
263
+ self._primary_disabled = True
264
+ except TtsTransientError as exc:
265
+ logger.warning(
266
+ "primary voice hiccup (%s); using the local voice for this line", exc
267
+ )
268
+ except Exception:
269
+ logger.exception("primary voice raised unexpectedly; using the local voice")
270
+ return self._fallback.synthesize(text)
271
+
272
+
273
+ def _resample_to_target(samples: NDArray[np.float32], source_rate_hz: int) -> NDArray[np.float32]:
274
+ """Resample mono audio onto ``TARGET_SAMPLE_RATE_HZ`` float32.
275
+
276
+ A pass-through when the rates already match, so the common Piper-at-16 kHz
277
+ case pays nothing. Otherwise polyphase resampling via ``scipy.signal`` β€”
278
+ imported lazily, matching the HAL adapters, so this module stays importable
279
+ without SciPy installed. Reducing up/down by their GCD keeps the polyphase
280
+ filter small (16 kHz from 22.05 kHz is 320/441, not 16000/22050).
281
+
282
+ Args:
283
+ samples: Mono float32 at ``source_rate_hz``.
284
+ source_rate_hz: The rate ``samples`` were produced at.
285
+
286
+ Returns:
287
+ Contiguous float32 at ``TARGET_SAMPLE_RATE_HZ``, clipped to [-1, 1]
288
+ because polyphase filtering can ring a hair past full scale.
289
+ """
290
+ if samples.ndim != 1:
291
+ raise ValueError(f"expected mono (1-D) audio, got shape {samples.shape}")
292
+ if source_rate_hz <= 0:
293
+ raise ValueError(f"source_rate_hz must be positive, got {source_rate_hz}")
294
+ if source_rate_hz == TARGET_SAMPLE_RATE_HZ or samples.size == 0:
295
+ return np.ascontiguousarray(samples, dtype=AUDIO_DTYPE)
296
+
297
+ from scipy.signal import resample_poly
298
+
299
+ divisor = gcd(source_rate_hz, TARGET_SAMPLE_RATE_HZ)
300
+ up = TARGET_SAMPLE_RATE_HZ // divisor
301
+ down = source_rate_hz // divisor
302
+ resampled = resample_poly(samples.astype(np.float64), up, down)
303
+ return np.ascontiguousarray(np.clip(resampled, -1.0, 1.0), dtype=AUDIO_DTYPE)
304
+
305
+
306
+ async def _stream_chunks(
307
+ samples: NDArray[np.float32], *, chunk_ms: int
308
+ ) -> AsyncIterator[AudioChunk]:
309
+ """Slice 16 kHz mono ``samples`` into ~``chunk_ms`` ``AudioChunk``s.
310
+
311
+ Timestamps advance by each chunk's own duration so the stream carries a
312
+ coherent playback clock. A short final chunk is yielded as-is rather than
313
+ padded β€” padding would inject silence the speaker would actually play.
314
+ """
315
+ samples_per_chunk = int(TARGET_SAMPLE_RATE_HZ * chunk_ms / 1000)
316
+ elapsed_s = 0.0
317
+ for start in range(0, len(samples), samples_per_chunk):
318
+ block = np.ascontiguousarray(samples[start : start + samples_per_chunk], dtype=AUDIO_DTYPE)
319
+ yield AudioChunk(
320
+ samples=block,
321
+ sample_rate_hz=TARGET_SAMPLE_RATE_HZ,
322
+ channels=1,
323
+ timestamp_s=elapsed_s,
324
+ )
325
+ elapsed_s += len(block) / TARGET_SAMPLE_RATE_HZ
326
+
327
+
328
+ @runtime_checkable
329
+ class SupportsAudioOut(Protocol):
330
+ """The slice of a HAL body that ``speak`` needs: a sink for an audio stream.
331
+
332
+ Typing ``speak`` to this narrow shape (not the full ``HALAdapter``) is what
333
+ lets the screen-avatar test double β€” which implements only ``audio_out`` β€”
334
+ stand in for the real Reachy body, and keeps ``speak`` honest about what it
335
+ actually touches.
336
+ """
337
+
338
+ async def audio_out(self, chunks: AsyncIterator[AudioChunk]) -> None:
339
+ """Play a stream of audio chunks, returning once they are consumed."""
340
+ ...
341
+
342
+
343
+ async def speak(
344
+ text: str,
345
+ body: SupportsAudioOut,
346
+ engine: TtsEngine,
347
+ *,
348
+ chunk_ms: int = DEFAULT_CHUNK_MS,
349
+ ) -> None:
350
+ """Synthesize ``text`` and stream it to ``body`` as 16 kHz float32 mono.
351
+
352
+ The full expression path for one line: synthesize with ``engine``, resample
353
+ to the body's expected rate, and stream ~``chunk_ms`` chunks into
354
+ ``body.audio_out`` so playback can start before synthesis finishes. Empty
355
+ or whitespace-only text streams nothing β€” there is no audio to play, and an
356
+ empty stream is the correct representation of silence.
357
+
358
+ Args:
359
+ text: The line to speak.
360
+ body: The HAL adapter that will play the audio.
361
+ engine: The synthesis backend.
362
+ chunk_ms: Playback chunk duration. Must be positive.
363
+ """
364
+ if chunk_ms <= 0:
365
+ raise ValueError(f"chunk_ms must be positive, got {chunk_ms}")
366
+
367
+ # Synthesis is a blocking call β€” a Piper decode on a slow CPU, or a network
368
+ # round-trip to a cloud engine β€” so it runs on a worker thread. On the event
369
+ # loop it would freeze everything (a still-speaking reply, motion, the stop
370
+ # check) for its whole duration; off it, the loop keeps serving them.
371
+ raw_samples, source_rate_hz = await asyncio.to_thread(engine.synthesize, text)
372
+ samples = _resample_to_target(raw_samples, source_rate_hz)
373
+ await body.audio_out(_stream_chunks(samples, chunk_ms=chunk_ms))
pyproject.toml CHANGED
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
 
6
  [project]
7
  name = "chittios"
8
- version = "1.1.0"
9
  description = "ChittiOS β€” a local-first household companion for Reachy Mini"
10
  readme = "README.md"
11
  requires-python = ">=3.10"
 
5
 
6
  [project]
7
  name = "chittios"
8
+ version = "1.2.0"
9
  description = "ChittiOS β€” a local-first household companion for Reachy Mini"
10
  readme = "README.md"
11
  requires-python = ">=3.10"