Spaces:
Running
Running
v1.2.0: optional ElevenLabs cloud voice (onboarding key box, local Piper fallback)
Browse files- chittios/main.py +790 -706
- chittios/static/index.html +16 -0
- chittios/static/main.js +66 -6
- chittios/static/style.css +35 -0
- chittios_core/expression/config.py +200 -0
- chittios_core/expression/elevenlabs.py +164 -0
- chittios_core/expression/tts.py +373 -305
- pyproject.toml +1 -1
chittios/main.py
CHANGED
|
@@ -1,706 +1,790 @@
|
|
| 1 |
-
"""ChittiOS β the Reachy Mini app shell (Phase 7: the composed OS).
|
| 2 |
-
|
| 3 |
-
Phase 1 proved the publish β install β run pipeline with a greeting and idle
|
| 4 |
-
breathing. This is the real app: a thin shell that constructs the *concrete*,
|
| 5 |
-
hardware- and model-backed engines and hands them to ``chittios_core``'s
|
| 6 |
-
:class:`~chittios_core.orchestrator.ChittiOS`, which owns every line of composition
|
| 7 |
-
logic. Nothing here decides *what* ChittiOS does β it only supplies the real body,
|
| 8 |
-
the real camera and microphone, and the real STT/LLM/TTS to the seams the
|
| 9 |
-
orchestrator drives, exactly where a test supplies stubs.
|
| 10 |
-
|
| 11 |
-
Thin, and gracefully degrading
|
| 12 |
-
------------------------------
|
| 13 |
-
Every heavy engine is optional: a fresh SBC may lack a Piper voice, a Whisper
|
| 14 |
-
model, or a reachable LLM, and the household should still get a robot that
|
| 15 |
-
recognises them and says hello rather than one that refuses to boot. So each
|
| 16 |
-
engine is built behind a small guarded factory that returns ``None`` (or a stub
|
| 17 |
-
fallback for the one engine β TTS β the greeting cannot do without) when its
|
| 18 |
-
dependency or device is absent. A missing LLM costs conversation and nothing
|
| 19 |
-
else; perception, greeting, and idle motion run regardless.
|
| 20 |
-
|
| 21 |
-
Idle breathing stays a raw-SDK control loop
|
| 22 |
-
--------------------------------------------
|
| 23 |
-
The "alive" breathing is the one thing that cannot go through the HAL motion
|
| 24 |
-
primitives: it needs ``set_target`` at a fixed 60 Hz (repeated ``goto_target``
|
| 25 |
-
each restart the interpolation β the judder Phase 1 diagnosed). So it stays here,
|
| 26 |
-
against the raw SDK, wrapped as an :class:`IdleMotion` the orchestrator runs on
|
| 27 |
-
its own thread β the reference control loop, unchanged, just handed to the
|
| 28 |
-
composition instead of being ``run``'s only body.
|
| 29 |
-
"""
|
| 30 |
-
|
| 31 |
-
from __future__ import annotations
|
| 32 |
-
|
| 33 |
-
import logging
|
| 34 |
-
import os
|
| 35 |
-
import threading
|
| 36 |
-
import time
|
| 37 |
-
from collections.abc import Callable
|
| 38 |
-
from contextlib import ExitStack
|
| 39 |
-
from datetime import UTC, datetime
|
| 40 |
-
from pathlib import Path
|
| 41 |
-
from typing import TypeVar
|
| 42 |
-
|
| 43 |
-
import numpy as np
|
| 44 |
-
from fastapi import FastAPI
|
| 45 |
-
from numpy.typing import NDArray
|
| 46 |
-
from pydantic import BaseModel
|
| 47 |
-
from reachy_mini import ReachyMini, ReachyMiniApp
|
| 48 |
-
from reachy_mini.utils import create_head_pose
|
| 49 |
-
|
| 50 |
-
from chittios_core.context import ContextManager
|
| 51 |
-
from chittios_core.conversation.config import LlmConfig, load_llm_config, write_llm_config
|
| 52 |
-
from chittios_core.conversation.llm import OpenAICompatibleLlm
|
| 53 |
-
from chittios_core.conversation.stt import WhisperStt
|
| 54 |
-
from chittios_core.conversation.vad import OnnxVadSegmenter
|
| 55 |
-
from chittios_core.expression.
|
| 56 |
-
from chittios_core.
|
| 57 |
-
from chittios_core.
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
)
|
| 77 |
-
from chittios_core.identity.onboarding.
|
| 78 |
-
from chittios_core.identity.
|
| 79 |
-
from chittios_core.identity.
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
from chittios_core.identity.
|
| 87 |
-
from chittios_core.identity.
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
from chittios_core.
|
| 97 |
-
from chittios_core.
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
)
|
| 103 |
-
from
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
#:
|
| 125 |
-
#:
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
#
|
| 131 |
-
|
| 132 |
-
#:
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
#
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
speech_out:
|
| 207 |
-
|
| 208 |
-
greeting:
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
self
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
"""
|
| 242 |
-
|
| 243 |
-
def read(self) -> None:
|
| 244 |
-
return None
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
class
|
| 248 |
-
"""A
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
a
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
return get_or_create_key(
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
model
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
#
|
| 416 |
-
#
|
| 417 |
-
#
|
| 418 |
-
#
|
| 419 |
-
#
|
| 420 |
-
#
|
| 421 |
-
|
| 422 |
-
#
|
| 423 |
-
#
|
| 424 |
-
#
|
| 425 |
-
#
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
#
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
return {
|
| 534 |
-
"base_url":
|
| 535 |
-
"model":
|
| 536 |
-
"has_api_key":
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
return
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
|
| 679 |
-
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
| 706 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ChittiOS β the Reachy Mini app shell (Phase 7: the composed OS).
|
| 2 |
+
|
| 3 |
+
Phase 1 proved the publish β install β run pipeline with a greeting and idle
|
| 4 |
+
breathing. This is the real app: a thin shell that constructs the *concrete*,
|
| 5 |
+
hardware- and model-backed engines and hands them to ``chittios_core``'s
|
| 6 |
+
:class:`~chittios_core.orchestrator.ChittiOS`, which owns every line of composition
|
| 7 |
+
logic. Nothing here decides *what* ChittiOS does β it only supplies the real body,
|
| 8 |
+
the real camera and microphone, and the real STT/LLM/TTS to the seams the
|
| 9 |
+
orchestrator drives, exactly where a test supplies stubs.
|
| 10 |
+
|
| 11 |
+
Thin, and gracefully degrading
|
| 12 |
+
------------------------------
|
| 13 |
+
Every heavy engine is optional: a fresh SBC may lack a Piper voice, a Whisper
|
| 14 |
+
model, or a reachable LLM, and the household should still get a robot that
|
| 15 |
+
recognises them and says hello rather than one that refuses to boot. So each
|
| 16 |
+
engine is built behind a small guarded factory that returns ``None`` (or a stub
|
| 17 |
+
fallback for the one engine β TTS β the greeting cannot do without) when its
|
| 18 |
+
dependency or device is absent. A missing LLM costs conversation and nothing
|
| 19 |
+
else; perception, greeting, and idle motion run regardless.
|
| 20 |
+
|
| 21 |
+
Idle breathing stays a raw-SDK control loop
|
| 22 |
+
--------------------------------------------
|
| 23 |
+
The "alive" breathing is the one thing that cannot go through the HAL motion
|
| 24 |
+
primitives: it needs ``set_target`` at a fixed 60 Hz (repeated ``goto_target``
|
| 25 |
+
each restart the interpolation β the judder Phase 1 diagnosed). So it stays here,
|
| 26 |
+
against the raw SDK, wrapped as an :class:`IdleMotion` the orchestrator runs on
|
| 27 |
+
its own thread β the reference control loop, unchanged, just handed to the
|
| 28 |
+
composition instead of being ``run``'s only body.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import logging
|
| 34 |
+
import os
|
| 35 |
+
import threading
|
| 36 |
+
import time
|
| 37 |
+
from collections.abc import Callable
|
| 38 |
+
from contextlib import ExitStack
|
| 39 |
+
from datetime import UTC, datetime
|
| 40 |
+
from pathlib import Path
|
| 41 |
+
from typing import TypeVar
|
| 42 |
+
|
| 43 |
+
import numpy as np
|
| 44 |
+
from fastapi import FastAPI
|
| 45 |
+
from numpy.typing import NDArray
|
| 46 |
+
from pydantic import BaseModel
|
| 47 |
+
from reachy_mini import ReachyMini, ReachyMiniApp
|
| 48 |
+
from reachy_mini.utils import create_head_pose
|
| 49 |
+
|
| 50 |
+
from chittios_core.context import ContextManager
|
| 51 |
+
from chittios_core.conversation.config import LlmConfig, load_llm_config, write_llm_config
|
| 52 |
+
from chittios_core.conversation.llm import OpenAICompatibleLlm
|
| 53 |
+
from chittios_core.conversation.stt import WhisperStt
|
| 54 |
+
from chittios_core.conversation.vad import OnnxVadSegmenter
|
| 55 |
+
from chittios_core.expression.config import VoiceConfig, load_voice_config, write_voice_config
|
| 56 |
+
from chittios_core.expression.elevenlabs import ElevenLabsTtsEngine
|
| 57 |
+
from chittios_core.expression.tts import (
|
| 58 |
+
FallbackTtsEngine,
|
| 59 |
+
PiperTtsEngine,
|
| 60 |
+
StubTtsEngine,
|
| 61 |
+
TtsEngine,
|
| 62 |
+
)
|
| 63 |
+
from chittios_core.identity import roster
|
| 64 |
+
from chittios_core.identity.face.detector import FaceDetectorProtocol, YuNetDetector
|
| 65 |
+
from chittios_core.identity.face.embedder import SFaceEmbedder
|
| 66 |
+
from chittios_core.identity.face.provider import FaceIdentityProvider
|
| 67 |
+
from chittios_core.identity.onboarding.adapters import (
|
| 68 |
+
AntennaHoldSignal,
|
| 69 |
+
MicNameRecorder,
|
| 70 |
+
ModelEnrollerFactory,
|
| 71 |
+
PassphraseCustody,
|
| 72 |
+
PullFaceSource,
|
| 73 |
+
PullVoiceSource,
|
| 74 |
+
SyncSpeechOut,
|
| 75 |
+
WhisperSpeechIn,
|
| 76 |
+
)
|
| 77 |
+
from chittios_core.identity.onboarding.authority import AuthorityGate
|
| 78 |
+
from chittios_core.identity.onboarding.conductor import CaptureConductor
|
| 79 |
+
from chittios_core.identity.onboarding.dialog import (
|
| 80 |
+
DialogOutcome,
|
| 81 |
+
DialogResult,
|
| 82 |
+
EnrollmentDialog,
|
| 83 |
+
)
|
| 84 |
+
from chittios_core.identity.onboarding.ports import SessionRecognizer, SpeechOut
|
| 85 |
+
from chittios_core.identity.session import Session
|
| 86 |
+
from chittios_core.identity.store.enrollments import EnrollmentStore
|
| 87 |
+
from chittios_core.identity.store.keys import (
|
| 88 |
+
FileKeyStore,
|
| 89 |
+
KeyCustodyError,
|
| 90 |
+
KeyringKeyStore,
|
| 91 |
+
get_or_create_key,
|
| 92 |
+
)
|
| 93 |
+
from chittios_core.identity.types import IdentityClaim, IdentityProvider, Modality
|
| 94 |
+
from chittios_core.identity.voice.embedder import ECAPAEmbedder
|
| 95 |
+
from chittios_core.identity.voice.provider import VoiceIdentityProvider
|
| 96 |
+
from chittios_core.identity.voice.segmenter import SegmenterProtocol, SpeechSegment
|
| 97 |
+
from chittios_core.orchestrator import (
|
| 98 |
+
ChittiOS,
|
| 99 |
+
ConversationSeams,
|
| 100 |
+
OwnerEnrollment,
|
| 101 |
+
PerceptionSeams,
|
| 102 |
+
)
|
| 103 |
+
from chittios_core.perception.loop import FacePipeline
|
| 104 |
+
from chittios_core.perception.sources import (
|
| 105 |
+
CameraSource,
|
| 106 |
+
MicSource,
|
| 107 |
+
ReachyMiniCameraSource,
|
| 108 |
+
SoundDeviceMicSource,
|
| 109 |
+
)
|
| 110 |
+
from hal.reachy_mini.adapter import ReachyMiniAdapter
|
| 111 |
+
|
| 112 |
+
logger = logging.getLogger(__name__)
|
| 113 |
+
|
| 114 |
+
_T = TypeVar("_T")
|
| 115 |
+
|
| 116 |
+
#: Where the enrolled household lives on the robot. The store creates its parent.
|
| 117 |
+
STORE_PATH = Path.home() / ".local" / "share" / "chittios" / "enrollments.db"
|
| 118 |
+
|
| 119 |
+
#: Where the recoverable passphrase custody vault lives β the Argon2id-wrapped
|
| 120 |
+
#: data key the recovery flow re-derives. Beside the store because a wrapped key
|
| 121 |
+
#: is ciphertext, useless without the household passphrase (see vault.py).
|
| 122 |
+
VAULT_PATH = Path.home() / ".local" / "share" / "chittios" / "passphrase.vault"
|
| 123 |
+
|
| 124 |
+
#: Piper voice model. Prefer the one bundled with the app so a fresh robot speaks
|
| 125 |
+
#: real words out of the box; fall back to a user-supplied model in the data dir.
|
| 126 |
+
_BUNDLED_VOICE = Path(__file__).parent / "assets" / "voice.onnx"
|
| 127 |
+
_USER_VOICE = Path.home() / ".local" / "share" / "chittios" / "voice.onnx"
|
| 128 |
+
|
| 129 |
+
#: Spoken by the empty-store fallback when the robot cannot run onboarding β
|
| 130 |
+
#: honest and warm rather than silent. The robot then idles (breathing) rather
|
| 131 |
+
#: than exiting, so a household that unboxes it into a half-provisioned state
|
| 132 |
+
#: still meets something alive.
|
| 133 |
+
ONBOARDING_UNAVAILABLE_GREETING = (
|
| 134 |
+
"Hello. I'm not quite ready to set up your household yet, but I'm here with you."
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
# ββ idle breathing (the reference 60 Hz control loop) ββββββββββββββββββββββ
|
| 138 |
+
|
| 139 |
+
#: Control-loop rate. The fleet app uses 60 Hz; the docs floor smoothness at
|
| 140 |
+
#: 30 Hz. One ``set_target`` per tick, period held by adaptive sleep.
|
| 141 |
+
CONTROL_FREQUENCY_HZ = 60.0
|
| 142 |
+
CONTROL_PERIOD_S = 1.0 / CONTROL_FREQUENCY_HZ
|
| 143 |
+
|
| 144 |
+
#: Idle "breathing": the target pose is a smooth sine of time (not discrete
|
| 145 |
+
#: commands). Amplitudes kept small and slow, matching the reference breathing.
|
| 146 |
+
BREATHING_PITCH_DEG = 3.0
|
| 147 |
+
BREATHING_PITCH_HZ = 0.2
|
| 148 |
+
LOOK_YAW_DEG = 8.0
|
| 149 |
+
LOOK_YAW_HZ = 0.05
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class BreathingMotion:
|
| 153 |
+
"""The idle "alive" motion, as an :class:`IdleMotion` the orchestrator runs.
|
| 154 |
+
|
| 155 |
+
The Phase 1 control loop, unchanged in substance: one ``set_target`` per tick
|
| 156 |
+
at a monotonic-aligned 60 Hz, the pose a smooth sine of time. It lives against
|
| 157 |
+
the raw SDK rather than the HAL because it needs ``set_target``'s immediacy β
|
| 158 |
+
the HAL's queued ``goto_target`` would restart interpolation every tick and
|
| 159 |
+
judder. The orchestrator runs this on its own thread and stops it with the same
|
| 160 |
+
event that stops everything else.
|
| 161 |
+
|
| 162 |
+
Args:
|
| 163 |
+
mini: The connected robot to drive.
|
| 164 |
+
"""
|
| 165 |
+
|
| 166 |
+
def __init__(self, mini: ReachyMini) -> None:
|
| 167 |
+
self._mini = mini
|
| 168 |
+
|
| 169 |
+
def run(self, stop: threading.Event) -> None:
|
| 170 |
+
"""Breathe until ``stop`` is set, then return."""
|
| 171 |
+
start = time.monotonic()
|
| 172 |
+
while not stop.is_set():
|
| 173 |
+
tick_start = time.monotonic()
|
| 174 |
+
elapsed = tick_start - start
|
| 175 |
+
|
| 176 |
+
pitch_deg = BREATHING_PITCH_DEG * np.sin(2.0 * np.pi * BREATHING_PITCH_HZ * elapsed)
|
| 177 |
+
yaw_deg = LOOK_YAW_DEG * np.sin(2.0 * np.pi * LOOK_YAW_HZ * elapsed)
|
| 178 |
+
self._mini.set_target(head=create_head_pose(pitch=pitch_deg, yaw=yaw_deg, degrees=True))
|
| 179 |
+
|
| 180 |
+
# Adaptive sleep holds the period regardless of per-tick cost; the
|
| 181 |
+
# event doubles as an interruptible sleep, so a stop returns at once.
|
| 182 |
+
sleep_s = max(0.0, CONTROL_PERIOD_S - (time.monotonic() - tick_start))
|
| 183 |
+
if stop.wait(sleep_s):
|
| 184 |
+
break
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
class _GreetAndIdleFallback:
|
| 188 |
+
"""The empty-store path when onboarding genuinely cannot be assembled.
|
| 189 |
+
|
| 190 |
+
An unboxed robot has an empty store and, normally, runs the spoken owner
|
| 191 |
+
enrollment. But if the pieces that ritual needs β a working microphone, the
|
| 192 |
+
voice models, the vault β cannot be built, the honest failure is not to exit
|
| 193 |
+
silently (a robot that does nothing looks broken) but to *say so* and stay
|
| 194 |
+
alive. Staying alive is the orchestrator's job now: it breathes on its own
|
| 195 |
+
thread and offers ``enroll_owner`` again on a cadence for the whole empty-store
|
| 196 |
+
lifetime, so this only owes a single greeting. It says its line the first time
|
| 197 |
+
it is offered and returns ``FAILED``; subsequent offers return ``FAILED``
|
| 198 |
+
silently, so the robot breathes on quietly rather than repeating itself.
|
| 199 |
+
|
| 200 |
+
It satisfies :class:`OwnerEnrollment` structurally, so the orchestrator drives
|
| 201 |
+
it through exactly the same ``enroll_owner`` seam it drives the real dialog
|
| 202 |
+
through β the orchestrator's own empty-store semantics are untouched; only
|
| 203 |
+
*what* is wired into that seam differs when the dialog could not be built.
|
| 204 |
+
|
| 205 |
+
Args:
|
| 206 |
+
speech_out: How the greeting is spoken (the same sync TTS seam the dialog
|
| 207 |
+
would have used).
|
| 208 |
+
greeting: The single line spoken on the first offer.
|
| 209 |
+
"""
|
| 210 |
+
|
| 211 |
+
def __init__(
|
| 212 |
+
self,
|
| 213 |
+
speech_out: SpeechOut,
|
| 214 |
+
*,
|
| 215 |
+
greeting: str = ONBOARDING_UNAVAILABLE_GREETING,
|
| 216 |
+
) -> None:
|
| 217 |
+
self._speech_out = speech_out
|
| 218 |
+
self._greeting = greeting
|
| 219 |
+
self._greeted = False
|
| 220 |
+
|
| 221 |
+
def enroll_owner(self) -> DialogResult:
|
| 222 |
+
"""Greet on the first offer, then stay quiet. Nothing is ever enrolled."""
|
| 223 |
+
if not self._greeted:
|
| 224 |
+
self._greeted = True
|
| 225 |
+
try:
|
| 226 |
+
self._speech_out.speak(self._greeting)
|
| 227 |
+
except Exception:
|
| 228 |
+
logger.warning("the fallback greeting could not be spoken", exc_info=True)
|
| 229 |
+
return DialogResult(DialogOutcome.FAILED)
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
# ββ null sources (degrade when a device is absent) βββββββββββββββββββββββββ
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
class _NullCamera:
|
| 236 |
+
"""A :class:`CameraSource` that always sees nothing.
|
| 237 |
+
|
| 238 |
+
Used when no camera can be opened: perception then runs on voice alone rather
|
| 239 |
+
than failing to start. A returned ``None`` is the source contract's ordinary
|
| 240 |
+
"nothing right now", so the loop simply never gets a face.
|
| 241 |
+
"""
|
| 242 |
+
|
| 243 |
+
def read(self) -> None:
|
| 244 |
+
return None
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
class _NullMic:
|
| 248 |
+
"""A :class:`MicSource` that always hears nothing β the audio counterpart."""
|
| 249 |
+
|
| 250 |
+
def read(self) -> None:
|
| 251 |
+
return None
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
class _NullVoiceProvider:
|
| 255 |
+
"""A voice :class:`IdentityProvider` that recognises no one β face-only builds.
|
| 256 |
+
|
| 257 |
+
The speaker encoder (ECAPA) is torch- and speechbrain-backed, ~2 GB that does
|
| 258 |
+
not fit on the Reachy Mini, so voice **identification** is dropped on the
|
| 259 |
+
robot (the owner's call). Perception still segments audio β the conversation
|
| 260 |
+
loop needs those utterances for STT β but the voice *provider* it drives is
|
| 261 |
+
this: every utterance returns an unknown claim, exactly as the real provider
|
| 262 |
+
does for anyone unenrolled, so the fusion and authorization paths downstream
|
| 263 |
+
are unchanged and simply never receive a voice match. Recognition runs on the
|
| 264 |
+
face alone.
|
| 265 |
+
|
| 266 |
+
It satisfies :class:`~chittios_core.identity.types.IdentityProvider`\\
|
| 267 |
+
``[SpeechSegment]`` structurally, so it drops into the same perception seam
|
| 268 |
+
the real provider fills, and it constructs nothing torch-backed.
|
| 269 |
+
"""
|
| 270 |
+
|
| 271 |
+
@property
|
| 272 |
+
def modality(self) -> Modality:
|
| 273 |
+
return Modality.VOICE
|
| 274 |
+
|
| 275 |
+
def identify(
|
| 276 |
+
self, observation: SpeechSegment, /, *, observed_at: datetime | None = None
|
| 277 |
+
) -> IdentityClaim:
|
| 278 |
+
"""Return an unknown voice claim for any utterance β no one is identified."""
|
| 279 |
+
return IdentityClaim.unknown(
|
| 280 |
+
modality=Modality.VOICE, observed_at=observed_at or datetime.now(UTC)
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
class _NullVoiceEmbedder:
|
| 285 |
+
"""A voice embedder that is never called β the placeholder for face-only sittings.
|
| 286 |
+
|
| 287 |
+
``ModelEnrollerFactory`` holds a voice embedder so it can mint voice enrollers,
|
| 288 |
+
but a face-only enrollment (``enroll_voice=False``) never asks it for one, so
|
| 289 |
+
this stand-in is only ever *held*, never *invoked*. It exists so the factory's
|
| 290 |
+
wiring is uniform whether or not a speaker encoder is present, without
|
| 291 |
+
constructing the torch-backed ECAPA on a robot that cannot run it. Being
|
| 292 |
+
called would mean a face-only sitting tried to embed voice after all, which is
|
| 293 |
+
a wiring bug worth failing loudly on rather than papering over with a vector.
|
| 294 |
+
"""
|
| 295 |
+
|
| 296 |
+
def embed(self, canonical_audio: NDArray[np.float32], /) -> NDArray[np.float32]:
|
| 297 |
+
raise NotImplementedError(
|
| 298 |
+
"voice enrollment is disabled on this build; _NullVoiceEmbedder must never be called"
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def _guard(what: str, build: Callable[[], _T]) -> _T | None:
|
| 303 |
+
"""Build an optional engine, degrading to ``None`` if its runtime is absent.
|
| 304 |
+
|
| 305 |
+
The one place the "a missing model must not stop the robot booting" rule is
|
| 306 |
+
enforced: any failure to construct β a missing package, an absent model file,
|
| 307 |
+
an unopenable device β is logged and turned into ``None`` rather than raised.
|
| 308 |
+
"""
|
| 309 |
+
try:
|
| 310 |
+
return build()
|
| 311 |
+
except Exception:
|
| 312 |
+
logger.warning("%s is unavailable; ChittiOS will run without it", what, exc_info=True)
|
| 313 |
+
return None
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def _resolve_store_key() -> bytes:
|
| 317 |
+
"""The enrollment store's encryption key: OS keychain where one exists, else a
|
| 318 |
+
mode-0600 key file on a headless Linux host.
|
| 319 |
+
|
| 320 |
+
The robot is headless Linux with no login keyring, so ``KeyringKeyStore``
|
| 321 |
+
raises there. ``FileKeyStore`` keeps the key in a 0600 file beside the store β
|
| 322 |
+
weaker than a keychain (the key sits near the data it protects), but on a box
|
| 323 |
+
with no keychain it is that or no at-rest encryption at all. Windows and macOS
|
| 324 |
+
keep the keychain; ``FileKeyStore`` is POSIX-only, so this only falls back on
|
| 325 |
+
a POSIX host.
|
| 326 |
+
"""
|
| 327 |
+
try:
|
| 328 |
+
return get_or_create_key(KeyringKeyStore())
|
| 329 |
+
except KeyCustodyError:
|
| 330 |
+
if os.name != "posix":
|
| 331 |
+
raise
|
| 332 |
+
key_dir = STORE_PATH.parent
|
| 333 |
+
key_dir.mkdir(parents=True, exist_ok=True)
|
| 334 |
+
logger.warning("no OS keychain available; using a 0600 key file in %s", key_dir)
|
| 335 |
+
return get_or_create_key(FileKeyStore(key_dir))
|
| 336 |
+
|
| 337 |
+
|
| 338 |
+
class LlmSettingsUpdate(BaseModel):
|
| 339 |
+
"""The dashboard's ``POST /llm-config`` body β the three settings, plus key semantics.
|
| 340 |
+
|
| 341 |
+
``api_key`` distinguishes three intents a single password field cannot
|
| 342 |
+
otherwise express: ``None`` (field left blank) leaves the stored key
|
| 343 |
+
untouched, an empty string clears it, and any other string sets it β so
|
| 344 |
+
changing the URL never silently drops a previously saved key.
|
| 345 |
+
|
| 346 |
+
Defined at module scope, not inside :meth:`Chittios._register_settings_routes`,
|
| 347 |
+
on purpose: this module uses ``from __future__ import annotations`` (PEP 563),
|
| 348 |
+
so the route handler's ``update: LlmSettingsUpdate`` annotation is a *string*
|
| 349 |
+
that FastAPI resolves with :func:`typing.get_type_hints` against the module's
|
| 350 |
+
globals. A method-local class is not in those globals, so the annotation
|
| 351 |
+
fails to resolve to this Pydantic model and FastAPI falls back to treating
|
| 352 |
+
``update`` as a required *query* parameter β which no JSON body can satisfy,
|
| 353 |
+
yielding ``422 Unprocessable Entity`` on every save. Keeping the model here,
|
| 354 |
+
where ``get_type_hints`` can find it, is what makes the body bind.
|
| 355 |
+
"""
|
| 356 |
+
|
| 357 |
+
base_url: str
|
| 358 |
+
model: str
|
| 359 |
+
api_key: str | None = None
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
class VoiceSettingsUpdate(BaseModel):
|
| 363 |
+
"""The dashboard's ``POST /voice-config`` body β just the ElevenLabs key.
|
| 364 |
+
|
| 365 |
+
The same three-way ``api_key`` contract as :class:`LlmSettingsUpdate`: ``None``
|
| 366 |
+
(a blank field) keeps the stored key, ``""`` clears it, and any other string
|
| 367 |
+
sets it β so a household never loses a saved key by revisiting the page. The
|
| 368 |
+
voice and model are not surfaced in the UI (a warm premade voice is the
|
| 369 |
+
default), so they are not part of this body. Defined at module scope for the
|
| 370 |
+
same PEP 563 reason spelled out on :class:`LlmSettingsUpdate`: FastAPI must be
|
| 371 |
+
able to resolve the annotation against the module globals for the JSON body to
|
| 372 |
+
bind instead of degrading to a 422.
|
| 373 |
+
"""
|
| 374 |
+
|
| 375 |
+
api_key: str | None = None
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
class Chittios(ReachyMiniApp): # type: ignore[misc] # reachy_mini ships no stubs; base resolves to Any
|
| 379 |
+
# A small settings web UI, served by the daemon on this URL, lets a household
|
| 380 |
+
# point ChittiOS at their own LLM endpoint from the robot dashboard. The base
|
| 381 |
+
# class builds ``self.settings_app`` (a FastAPI app) from this URL; we add the
|
| 382 |
+
# config routes in ``run``. ``0.0.0.0`` so the dashboard reaches it over the LAN.
|
| 383 |
+
custom_app_url: str | None = "http://0.0.0.0:8042"
|
| 384 |
+
request_media_backend: str | None = None
|
| 385 |
+
|
| 386 |
+
def __init__(self, running_on_wireless: bool = False) -> None:
|
| 387 |
+
"""Build the app and register the settings routes before the server serves.
|
| 388 |
+
|
| 389 |
+
The base class builds ``self.settings_app`` here and ``wrapped_run`` starts
|
| 390 |
+
its web-server thread *before* it calls :meth:`run`. Registering the config
|
| 391 |
+
routes inside ``run`` (as this once did) therefore leaves a window β the
|
| 392 |
+
whole time the robot spends connecting to the daemon before ``run`` begins β
|
| 393 |
+
during which the server is already accepting requests but ``/llm-config``
|
| 394 |
+
does not exist yet, so the dashboard's on-load fetch races ahead of the
|
| 395 |
+
route and gets a 404. Registering them now, at construction, closes that
|
| 396 |
+
window: the routes are on the app before the first request can arrive.
|
| 397 |
+
"""
|
| 398 |
+
super().__init__(running_on_wireless)
|
| 399 |
+
self._register_settings_routes()
|
| 400 |
+
|
| 401 |
+
def run(self, reachy_mini: ReachyMini, stop_event: threading.Event) -> None:
|
| 402 |
+
"""Construct the real engines and hand them to the orchestrator.
|
| 403 |
+
|
| 404 |
+
Opens the camera and microphone for the app's lifetime (released on
|
| 405 |
+
return), builds whatever recognition, conversation, and speech engines the
|
| 406 |
+
machine can provide, and lets :class:`ChittiOS` compose and run them until
|
| 407 |
+
the daemon sets ``stop_event``.
|
| 408 |
+
"""
|
| 409 |
+
logging.basicConfig(level=logging.INFO)
|
| 410 |
+
body = ReachyMiniAdapter(reachy_mini)
|
| 411 |
+
store = EnrollmentStore(STORE_PATH, _resolve_store_key())
|
| 412 |
+
tts = self._build_tts()
|
| 413 |
+
|
| 414 |
+
with ExitStack() as sources:
|
| 415 |
+
# The camera and microphone are shared: on an empty store only
|
| 416 |
+
# onboarding reads them, on a populated one only perception does, and
|
| 417 |
+
# the two paths never run together β so one source each, not two
|
| 418 |
+
# fighting over an exclusive device.
|
| 419 |
+
#
|
| 420 |
+
# The camera comes from the SDK, not OpenCV: on the robot the daemon
|
| 421 |
+
# owns the Orbbec and streams frames over a GStreamer IPC endpoint, so
|
| 422 |
+
# a direct ``cv2.VideoCapture(0)`` fails ("could not open camera 0").
|
| 423 |
+
# ``reachy_mini.media.get_frame()`` is how an app gets frames while the
|
| 424 |
+
# daemon holds the device, and it already returns exactly the BGR
|
| 425 |
+
# frames the perception loop expects.
|
| 426 |
+
camera: CameraSource = (
|
| 427 |
+
_guard("camera", lambda: ReachyMiniCameraSource(reachy_mini.media)) or _NullCamera()
|
| 428 |
+
)
|
| 429 |
+
mic: MicSource = (
|
| 430 |
+
_guard("microphone", lambda: sources.enter_context(SoundDeviceMicSource()))
|
| 431 |
+
or _NullMic()
|
| 432 |
+
)
|
| 433 |
+
# The heavy recognition models, built once and shared by perception and
|
| 434 |
+
# onboarding so neither boot path pays to load them twice.
|
| 435 |
+
detector = YuNetDetector()
|
| 436 |
+
segmenter = OnnxVadSegmenter()
|
| 437 |
+
face_embedder = SFaceEmbedder()
|
| 438 |
+
# Voice identification is the one torch-backed engine left on this
|
| 439 |
+
# path: ECAPA pulls speechbrain and torch (~2 GB), which do not fit the
|
| 440 |
+
# Reachy Mini. Behind _guard it degrades to None when they are absent β
|
| 441 |
+
# recognition then runs on the face alone (_NullVoiceProvider) and the
|
| 442 |
+
# first-boot sitting enrols face-only. On a dev box with the `voice`
|
| 443 |
+
# extra installed it builds as before and voice-ID is fully live.
|
| 444 |
+
voice_embedder = _guard("voice identification (ECAPA)", ECAPAEmbedder)
|
| 445 |
+
# ``tiny.en`` rather than ``base``: on this SBC's CPU, ``base`` spends
|
| 446 |
+
# ~12 s decoding a 2 s turn, so utterances queue faster than they clear
|
| 447 |
+
# and a reply lands a minute late β unusable. The English-only ``tiny``
|
| 448 |
+
# is several times faster for a modest accuracy cost that short
|
| 449 |
+
# household turns tolerate well, keeping time-to-answer conversational.
|
| 450 |
+
#
|
| 451 |
+
# The ``.en`` build also *is* the language lock: forcing English keeps
|
| 452 |
+
# faster-whisper from auto-detecting per utterance, which on short,
|
| 453 |
+
# quiet turns β a name, a yes β has mis-fired to Italian on the robot
|
| 454 |
+
# and garbled every transcript. ``language="en"`` is kept as belt and
|
| 455 |
+
# braces. The core WhisperStt default stays language-agnostic; only the
|
| 456 |
+
# app pins the model and the language ChittiOS speaks.
|
| 457 |
+
stt = _guard("Whisper STT", lambda: WhisperStt("tiny.en", language="en"))
|
| 458 |
+
|
| 459 |
+
perception = self._build_perception(
|
| 460 |
+
store, camera, mic, detector, segmenter, face_embedder, voice_embedder
|
| 461 |
+
)
|
| 462 |
+
conversation = ConversationSeams(stt=stt, llm=_guard("LLM", self._build_llm))
|
| 463 |
+
onboarding = self._build_onboarding(
|
| 464 |
+
store=store,
|
| 465 |
+
body=body,
|
| 466 |
+
tts=tts,
|
| 467 |
+
reachy_mini=reachy_mini,
|
| 468 |
+
camera=camera,
|
| 469 |
+
mic=mic,
|
| 470 |
+
detector=detector,
|
| 471 |
+
segmenter=segmenter,
|
| 472 |
+
face_embedder=face_embedder,
|
| 473 |
+
voice_embedder=voice_embedder,
|
| 474 |
+
stt=stt,
|
| 475 |
+
stop_event=stop_event,
|
| 476 |
+
)
|
| 477 |
+
|
| 478 |
+
orchestrator = ChittiOS(
|
| 479 |
+
store=store,
|
| 480 |
+
perception=perception,
|
| 481 |
+
conversation=conversation,
|
| 482 |
+
tts=tts,
|
| 483 |
+
context=ContextManager(),
|
| 484 |
+
idle_motion=BreathingMotion(reachy_mini),
|
| 485 |
+
onboarding=onboarding,
|
| 486 |
+
)
|
| 487 |
+
orchestrator.run(body, stop_event=stop_event)
|
| 488 |
+
|
| 489 |
+
body.close()
|
| 490 |
+
|
| 491 |
+
def _build_llm(self) -> OpenAICompatibleLlm:
|
| 492 |
+
"""Construct the conversation LLM from the household's resolved config.
|
| 493 |
+
|
| 494 |
+
The endpoint is not hardcoded: :func:`load_llm_config` resolves it from
|
| 495 |
+
the environment, then the config file the settings UI writes, then the
|
| 496 |
+
built-in default (the Spark's Gemma). Whatever comes back is where this
|
| 497 |
+
robot sends conversation transcripts β local by default, or wherever the
|
| 498 |
+
household pointed it. Built behind ``_guard`` in :meth:`run`, so a
|
| 499 |
+
malformed config or missing HTTP stack costs conversation and nothing else.
|
| 500 |
+
"""
|
| 501 |
+
config = load_llm_config()
|
| 502 |
+
logger.info("LLM endpoint: %s (model %s)", config.base_url, config.model)
|
| 503 |
+
return OpenAICompatibleLlm(
|
| 504 |
+
base_url=config.base_url,
|
| 505 |
+
model=config.model,
|
| 506 |
+
api_key=config.api_key,
|
| 507 |
+
)
|
| 508 |
+
|
| 509 |
+
def _register_settings_routes(self) -> None:
|
| 510 |
+
"""Add the LLM-endpoint settings routes to the dashboard's settings app.
|
| 511 |
+
|
| 512 |
+
The base class builds ``self.settings_app`` from ``custom_app_url`` and
|
| 513 |
+
serves ``static/index.html`` as the dashboard page; here we give that page
|
| 514 |
+
an API. ``GET /llm-config`` reports the resolved endpoint and whether a key
|
| 515 |
+
is set (never the key itself β a secret is not echoed back over HTTP);
|
| 516 |
+
``POST /llm-config`` writes the household's choice to the config file.
|
| 517 |
+
|
| 518 |
+
Writes take effect on the next app start β there is no live client swap, by
|
| 519 |
+
design: rebuilding the conversation engine mid-turn is far more surface
|
| 520 |
+
area than the feature needs, so the contract is simply "restart to apply".
|
| 521 |
+
"""
|
| 522 |
+
# The base class exposes ``settings_app`` untyped (reachy_mini ships no
|
| 523 |
+
# stubs); bind it to a concrete ``FastAPI`` local so its route decorators
|
| 524 |
+
# are typed rather than ``Any`` (which mypy rejects as an untyped decorator).
|
| 525 |
+
settings_app: FastAPI | None = self.settings_app
|
| 526 |
+
if settings_app is None: # webserver disabled; nothing to wire.
|
| 527 |
+
return
|
| 528 |
+
|
| 529 |
+
@settings_app.get("/llm-config")
|
| 530 |
+
def get_llm_config() -> dict[str, object]:
|
| 531 |
+
"""Report the resolved endpoint for the dashboard to display."""
|
| 532 |
+
config = load_llm_config()
|
| 533 |
+
return {
|
| 534 |
+
"base_url": config.base_url,
|
| 535 |
+
"model": config.model,
|
| 536 |
+
"has_api_key": config.api_key is not None,
|
| 537 |
+
}
|
| 538 |
+
|
| 539 |
+
@settings_app.post("/llm-config")
|
| 540 |
+
def set_llm_config(update: LlmSettingsUpdate) -> dict[str, object]:
|
| 541 |
+
"""Persist the household's endpoint choice; applied on next start."""
|
| 542 |
+
current = load_llm_config()
|
| 543 |
+
if update.api_key is None:
|
| 544 |
+
api_key = current.api_key # blank field: keep the stored key.
|
| 545 |
+
elif update.api_key == "":
|
| 546 |
+
api_key = None # explicit clear.
|
| 547 |
+
else:
|
| 548 |
+
api_key = update.api_key # a new key.
|
| 549 |
+
new_config = LlmConfig(
|
| 550 |
+
base_url=update.base_url.strip(),
|
| 551 |
+
model=update.model.strip(),
|
| 552 |
+
api_key=api_key,
|
| 553 |
+
)
|
| 554 |
+
write_llm_config(new_config)
|
| 555 |
+
logger.info("LLM settings saved; restart ChittiOS to apply")
|
| 556 |
+
return {
|
| 557 |
+
"base_url": new_config.base_url,
|
| 558 |
+
"model": new_config.model,
|
| 559 |
+
"has_api_key": new_config.api_key is not None,
|
| 560 |
+
"restart_required": True,
|
| 561 |
+
}
|
| 562 |
+
|
| 563 |
+
@settings_app.get("/voice-config")
|
| 564 |
+
def get_voice_config() -> dict[str, object]:
|
| 565 |
+
"""Report whether a cloud voice is configured β never the key itself.
|
| 566 |
+
|
| 567 |
+
The dashboard uses this to decide whether to show the key box at all:
|
| 568 |
+
once a key is stored it hides the field, so onboarding asks once and
|
| 569 |
+
never again.
|
| 570 |
+
"""
|
| 571 |
+
config = load_voice_config()
|
| 572 |
+
return {"has_api_key": config.api_key is not None}
|
| 573 |
+
|
| 574 |
+
@settings_app.post("/voice-config")
|
| 575 |
+
def set_voice_config(update: VoiceSettingsUpdate) -> dict[str, object]:
|
| 576 |
+
"""Persist the household's ElevenLabs key; applied on next start."""
|
| 577 |
+
current = load_voice_config()
|
| 578 |
+
if update.api_key is None:
|
| 579 |
+
api_key = current.api_key # blank field: keep the stored key.
|
| 580 |
+
elif update.api_key.strip() == "":
|
| 581 |
+
api_key = None # explicit clear (empty, or whitespace only).
|
| 582 |
+
else:
|
| 583 |
+
api_key = update.api_key.strip() # a new key.
|
| 584 |
+
new_config = VoiceConfig(
|
| 585 |
+
api_key=api_key,
|
| 586 |
+
voice_id=current.voice_id,
|
| 587 |
+
model_id=current.model_id,
|
| 588 |
+
)
|
| 589 |
+
write_voice_config(new_config)
|
| 590 |
+
logger.info("voice settings saved; restart ChittiOS to apply")
|
| 591 |
+
return {"has_api_key": new_config.api_key is not None, "restart_required": True}
|
| 592 |
+
|
| 593 |
+
def _build_perception(
|
| 594 |
+
self,
|
| 595 |
+
store: EnrollmentStore,
|
| 596 |
+
camera: CameraSource,
|
| 597 |
+
mic: MicSource,
|
| 598 |
+
detector: FaceDetectorProtocol,
|
| 599 |
+
segmenter: SegmenterProtocol,
|
| 600 |
+
face_embedder: SFaceEmbedder,
|
| 601 |
+
voice_embedder: ECAPAEmbedder | None,
|
| 602 |
+
) -> PerceptionSeams:
|
| 603 |
+
"""Assemble the perception seams from the shared devices and models.
|
| 604 |
+
|
| 605 |
+
The recognition providers are built from the roster loaded out of the
|
| 606 |
+
store, so the household the robot recognises is exactly the one enrolled.
|
| 607 |
+
The camera and microphone arrive already opened (or as silent null
|
| 608 |
+
sources when a device could not be opened), so this only wires the
|
| 609 |
+
pipelines around them. When no speaker embedder is present (the robot,
|
| 610 |
+
where ECAPA does not fit) the voice provider degrades to a
|
| 611 |
+
:class:`_NullVoiceProvider` so recognition runs on the face alone.
|
| 612 |
+
"""
|
| 613 |
+
face_provider = FaceIdentityProvider(face_embedder, roster.face_roster(store))
|
| 614 |
+
voice_provider: IdentityProvider[SpeechSegment] = (
|
| 615 |
+
VoiceIdentityProvider(voice_embedder, roster.voice_roster(store))
|
| 616 |
+
if voice_embedder is not None
|
| 617 |
+
else _NullVoiceProvider()
|
| 618 |
+
)
|
| 619 |
+
|
| 620 |
+
return PerceptionSeams(
|
| 621 |
+
face=FacePipeline(source=camera, detector=detector, provider=face_provider),
|
| 622 |
+
mic=mic,
|
| 623 |
+
segmenter=segmenter,
|
| 624 |
+
voice_provider=voice_provider,
|
| 625 |
+
)
|
| 626 |
+
|
| 627 |
+
def _build_onboarding(
|
| 628 |
+
self,
|
| 629 |
+
*,
|
| 630 |
+
store: EnrollmentStore,
|
| 631 |
+
body: ReachyMiniAdapter,
|
| 632 |
+
tts: TtsEngine,
|
| 633 |
+
reachy_mini: ReachyMini,
|
| 634 |
+
camera: CameraSource,
|
| 635 |
+
mic: MicSource,
|
| 636 |
+
detector: FaceDetectorProtocol,
|
| 637 |
+
segmenter: SegmenterProtocol,
|
| 638 |
+
face_embedder: SFaceEmbedder,
|
| 639 |
+
voice_embedder: ECAPAEmbedder | None,
|
| 640 |
+
stt: WhisperStt | None,
|
| 641 |
+
stop_event: threading.Event,
|
| 642 |
+
) -> OwnerEnrollment:
|
| 643 |
+
"""The first-boot ritual, wired from the shared seams β or a friendly fallback.
|
| 644 |
+
|
| 645 |
+
The whole spoken :class:`EnrollmentDialog` is assembled behind ``_guard``:
|
| 646 |
+
if any of its concrete ports cannot be built β no STT model, no keychain,
|
| 647 |
+
an unreadable device β the assembly degrades to :class:`_GreetAndIdleFallback`
|
| 648 |
+
rather than raising, so an empty store never leaves the robot exiting in
|
| 649 |
+
silence. The orchestrator drives whichever object comes back through the
|
| 650 |
+
same ``enroll_owner`` seam.
|
| 651 |
+
"""
|
| 652 |
+
speech_out = SyncSpeechOut(body, tts)
|
| 653 |
+
dialog = _guard(
|
| 654 |
+
"voice onboarding",
|
| 655 |
+
lambda: self._assemble_dialog(
|
| 656 |
+
store=store,
|
| 657 |
+
speech_out=speech_out,
|
| 658 |
+
reachy_mini=reachy_mini,
|
| 659 |
+
camera=camera,
|
| 660 |
+
mic=mic,
|
| 661 |
+
detector=detector,
|
| 662 |
+
segmenter=segmenter,
|
| 663 |
+
face_embedder=face_embedder,
|
| 664 |
+
voice_embedder=voice_embedder,
|
| 665 |
+
stt=stt,
|
| 666 |
+
stop_event=stop_event,
|
| 667 |
+
),
|
| 668 |
+
)
|
| 669 |
+
if dialog is not None:
|
| 670 |
+
return dialog
|
| 671 |
+
|
| 672 |
+
logger.info("voice onboarding is unavailable; the empty-store path will greet and idle")
|
| 673 |
+
return _GreetAndIdleFallback(speech_out)
|
| 674 |
+
|
| 675 |
+
def _assemble_dialog(
|
| 676 |
+
self,
|
| 677 |
+
*,
|
| 678 |
+
store: EnrollmentStore,
|
| 679 |
+
speech_out: SpeechOut,
|
| 680 |
+
reachy_mini: ReachyMini,
|
| 681 |
+
camera: CameraSource,
|
| 682 |
+
mic: MicSource,
|
| 683 |
+
detector: FaceDetectorProtocol,
|
| 684 |
+
segmenter: SegmenterProtocol,
|
| 685 |
+
face_embedder: SFaceEmbedder,
|
| 686 |
+
voice_embedder: ECAPAEmbedder | None,
|
| 687 |
+
stt: WhisperStt | None,
|
| 688 |
+
stop_event: threading.Event,
|
| 689 |
+
) -> EnrollmentDialog:
|
| 690 |
+
"""Wire the concrete ports into the spoken enrollment dialog.
|
| 691 |
+
|
| 692 |
+
Raises if any hard requirement is missing (STT is the one that can be
|
| 693 |
+
absent on a fresh install); the caller guards this, turning that raise
|
| 694 |
+
into the greet-and-idle fallback.
|
| 695 |
+
|
| 696 |
+
Voice *enrollment* follows voice identification: with no speaker embedder
|
| 697 |
+
(the robot) the sitting runs face-only β ``enroll_voice=False``, and the
|
| 698 |
+
factory holds a never-called :class:`_NullVoiceEmbedder` so its wiring is
|
| 699 |
+
uniform. Name and face capture, the antenna gate, and the passphrase
|
| 700 |
+
custody are unchanged, so a household is still fully enrolled by voice on
|
| 701 |
+
the robot; only the biometric *voice print* is omitted.
|
| 702 |
+
|
| 703 |
+
The owner-hold wait is wired to ``stop_event`` (``should_abort``) so a
|
| 704 |
+
daemon shutdown while the robot is waiting for the antenna hold is honoured
|
| 705 |
+
promptly rather than only after the wait's timeout elapses.
|
| 706 |
+
"""
|
| 707 |
+
if stt is None:
|
| 708 |
+
raise RuntimeError("voice onboarding needs Whisper STT to hear names and answers")
|
| 709 |
+
|
| 710 |
+
# On first boot the identity runtime does not exist yet, so recognition
|
| 711 |
+
# reports a guest (no one recognised) β correct for the owner bootstrap,
|
| 712 |
+
# which is gated on the antenna hold, not on a vouch.
|
| 713 |
+
recognizer = SessionRecognizer(Session.guest)
|
| 714 |
+
enroll_voice = voice_embedder is not None
|
| 715 |
+
return EnrollmentDialog(
|
| 716 |
+
speech_out=speech_out,
|
| 717 |
+
speech_in=WhisperSpeechIn(mic, segmenter, stt),
|
| 718 |
+
name_recorder=MicNameRecorder(mic, segmenter),
|
| 719 |
+
recognizer=recognizer,
|
| 720 |
+
gate=AuthorityGate(AntennaHoldSignal(reachy_mini), recognizer),
|
| 721 |
+
conductor=CaptureConductor(speech_out),
|
| 722 |
+
custody=PassphraseCustody(VAULT_PATH),
|
| 723 |
+
store=store,
|
| 724 |
+
face_source=PullFaceSource(camera, detector),
|
| 725 |
+
voice_source=PullVoiceSource(mic, segmenter),
|
| 726 |
+
enrollers=ModelEnrollerFactory(face_embedder, voice_embedder or _NullVoiceEmbedder()),
|
| 727 |
+
should_abort=stop_event.is_set,
|
| 728 |
+
enroll_voice=enroll_voice,
|
| 729 |
+
# No spoken setup passphrase: it only fed the (fragile, STT-based)
|
| 730 |
+
# recovery vault; the store is keyed by the on-device FileKeyStore.
|
| 731 |
+
# Setup is antenna hold -> face + name -> done.
|
| 732 |
+
require_passphrase=False,
|
| 733 |
+
)
|
| 734 |
+
|
| 735 |
+
def _build_tts(self) -> TtsEngine:
|
| 736 |
+
"""The speech engine: the household's cloud voice when configured, else local.
|
| 737 |
+
|
| 738 |
+
A configured ElevenLabs key (from ``ELEVENLABS_API_KEY`` or the settings
|
| 739 |
+
file the onboarding box writes) builds the cloud engine, wrapped so a
|
| 740 |
+
failure β a bad key, an exhausted quota, a dropped connection β falls back
|
| 741 |
+
to the local voice rather than muting the robot. No key means the local
|
| 742 |
+
voice outright. The local engine is always built first, precisely so it is
|
| 743 |
+
there to be the fallback.
|
| 744 |
+
"""
|
| 745 |
+
local = self._build_local_tts()
|
| 746 |
+
voice = load_voice_config()
|
| 747 |
+
key = voice.api_key
|
| 748 |
+
if key is not None:
|
| 749 |
+
cloud = _guard(
|
| 750 |
+
"ElevenLabs TTS",
|
| 751 |
+
lambda: ElevenLabsTtsEngine(
|
| 752 |
+
key, voice_id=voice.voice_id, model_id=voice.model_id
|
| 753 |
+
),
|
| 754 |
+
)
|
| 755 |
+
if cloud is not None:
|
| 756 |
+
logger.info(
|
| 757 |
+
"ElevenLabs voice enabled (voice %s, model %s); local voice is the fallback",
|
| 758 |
+
voice.voice_id,
|
| 759 |
+
voice.model_id,
|
| 760 |
+
)
|
| 761 |
+
return FallbackTtsEngine(cloud, fallback=local)
|
| 762 |
+
return local
|
| 763 |
+
|
| 764 |
+
def _build_local_tts(self) -> TtsEngine:
|
| 765 |
+
"""The local speech engine β Piper when a voice model is present, else the stub.
|
| 766 |
+
|
| 767 |
+
TTS is the one engine the orchestrator cannot do without: greeting has to
|
| 768 |
+
have a voice. So a missing Piper model degrades to the deterministic tone
|
| 769 |
+
rather than to ``None`` β a plain-spoken robot instead of a mute one. This
|
| 770 |
+
is also the fallback the cloud voice degrades to, so it must never itself
|
| 771 |
+
return ``None``.
|
| 772 |
+
"""
|
| 773 |
+
model = next((p for p in (_BUNDLED_VOICE, _USER_VOICE) if p.exists()), None)
|
| 774 |
+
if model is not None:
|
| 775 |
+
engine = _guard(
|
| 776 |
+
"Piper TTS",
|
| 777 |
+
lambda: PiperTtsEngine(str(model), config_path=f"{model}.json"),
|
| 778 |
+
)
|
| 779 |
+
if engine is not None:
|
| 780 |
+
return engine
|
| 781 |
+
logger.info("no Piper voice model available; using the stub tone voice")
|
| 782 |
+
return StubTtsEngine()
|
| 783 |
+
|
| 784 |
+
|
| 785 |
+
if __name__ == "__main__":
|
| 786 |
+
app = Chittios()
|
| 787 |
+
try:
|
| 788 |
+
app.wrapped_run()
|
| 789 |
+
except KeyboardInterrupt:
|
| 790 |
+
app.stop()
|
chittios/static/index.html
CHANGED
|
@@ -46,6 +46,22 @@
|
|
| 46 |
</label>
|
| 47 |
</label>
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
<div class="actions">
|
| 50 |
<button id="save-btn" type="submit">Save settings</button>
|
| 51 |
<span id="status" class="status" role="status" aria-live="polite"></span>
|
|
|
|
| 46 |
</label>
|
| 47 |
</label>
|
| 48 |
|
| 49 |
+
<div class="voice" id="voice-section">
|
| 50 |
+
<span class="label">Premium voice <span class="opt">optional</span></span>
|
| 51 |
+
<div id="voice-input-wrap">
|
| 52 |
+
<input id="eleven-key" name="eleven_key" type="password" autocomplete="off"
|
| 53 |
+
placeholder="Paste your ElevenLabs API key">
|
| 54 |
+
<span class="hint">Gives ChittiOS a natural cloud voice (ElevenLabs) instead of the
|
| 55 |
+
built-in one. Only the robot's <em>spoken reply</em> is sent for synthesis β
|
| 56 |
+
never your family's microphone audio, which always stays on the device. Leave
|
| 57 |
+
blank to keep the local voice.</span>
|
| 58 |
+
</div>
|
| 59 |
+
<p class="voice-on" id="voice-enabled" hidden>
|
| 60 |
+
β ElevenLabs voice enabled.
|
| 61 |
+
<button type="button" id="replace-voice" class="linkish">Replace key</button>
|
| 62 |
+
</p>
|
| 63 |
+
</div>
|
| 64 |
+
|
| 65 |
<div class="actions">
|
| 66 |
<button id="save-btn" type="submit">Save settings</button>
|
| 67 |
<span id="status" class="status" role="status" aria-live="polite"></span>
|
chittios/static/main.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
| 1 |
-
// The ChittiOS settings page: read the current LLM endpoint
|
| 2 |
-
//
|
| 3 |
-
//
|
| 4 |
-
//
|
| 5 |
-
//
|
|
|
|
|
|
|
| 6 |
|
| 7 |
const CONFIG_ENDPOINT = "/llm-config";
|
|
|
|
| 8 |
|
| 9 |
const els = {
|
| 10 |
form: document.getElementById("llm-form"),
|
|
@@ -15,6 +18,10 @@ const els = {
|
|
| 15 |
keyHint: document.getElementById("key-hint"),
|
| 16 |
saveBtn: document.getElementById("save-btn"),
|
| 17 |
status: document.getElementById("status"),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
};
|
| 19 |
|
| 20 |
function setStatus(message, kind) {
|
|
@@ -22,6 +29,8 @@ function setStatus(message, kind) {
|
|
| 22 |
els.status.className = kind ? `status ${kind}` : "status";
|
| 23 |
}
|
| 24 |
|
|
|
|
|
|
|
| 25 |
// Reflect whether a key is currently stored, without ever revealing it.
|
| 26 |
function reflectKeyState(hasApiKey) {
|
| 27 |
els.apiKey.placeholder = hasApiKey
|
|
@@ -56,6 +65,48 @@ function resolveApiKey() {
|
|
| 56 |
return typed.length > 0 ? typed : null;
|
| 57 |
}
|
| 58 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
async function saveConfig(event) {
|
| 60 |
event.preventDefault();
|
| 61 |
|
|
@@ -79,14 +130,23 @@ async function saveConfig(event) {
|
|
| 79 |
reflectKeyState(Boolean(data.has_api_key));
|
| 80 |
els.apiKey.value = "";
|
| 81 |
els.clearKey.checked = false;
|
|
|
|
|
|
|
|
|
|
| 82 |
setStatus("Saved. Restart ChittiOS to apply.", "ok");
|
| 83 |
} catch (e) {
|
| 84 |
setStatus("Could not save settings.", "err");
|
| 85 |
-
console.error("Failed to save
|
| 86 |
} finally {
|
| 87 |
els.saveBtn.disabled = false;
|
| 88 |
}
|
| 89 |
}
|
| 90 |
|
| 91 |
els.form.addEventListener("submit", saveConfig);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
loadConfig();
|
|
|
|
|
|
| 1 |
+
// The ChittiOS settings page: read the current LLM endpoint and (optionally) an
|
| 2 |
+
// ElevenLabs voice key, let the household change them, and write them back.
|
| 3 |
+
//
|
| 4 |
+
// Neither secret is ever pre-filled. The server reports only *whether* each key
|
| 5 |
+
// is set. The LLM key field stays visible so it can be changed; the ElevenLabs
|
| 6 |
+
// key box, by contrast, is shown until a key is stored and then hidden β onboard
|
| 7 |
+
// once, don't ask again β with a "Replace key" link for the rare rotation.
|
| 8 |
|
| 9 |
const CONFIG_ENDPOINT = "/llm-config";
|
| 10 |
+
const VOICE_ENDPOINT = "/voice-config";
|
| 11 |
|
| 12 |
const els = {
|
| 13 |
form: document.getElementById("llm-form"),
|
|
|
|
| 18 |
keyHint: document.getElementById("key-hint"),
|
| 19 |
saveBtn: document.getElementById("save-btn"),
|
| 20 |
status: document.getElementById("status"),
|
| 21 |
+
elevenKey: document.getElementById("eleven-key"),
|
| 22 |
+
voiceInputWrap: document.getElementById("voice-input-wrap"),
|
| 23 |
+
voiceEnabled: document.getElementById("voice-enabled"),
|
| 24 |
+
replaceVoice: document.getElementById("replace-voice"),
|
| 25 |
};
|
| 26 |
|
| 27 |
function setStatus(message, kind) {
|
|
|
|
| 29 |
els.status.className = kind ? `status ${kind}` : "status";
|
| 30 |
}
|
| 31 |
|
| 32 |
+
// ββ LLM endpoint βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 33 |
+
|
| 34 |
// Reflect whether a key is currently stored, without ever revealing it.
|
| 35 |
function reflectKeyState(hasApiKey) {
|
| 36 |
els.apiKey.placeholder = hasApiKey
|
|
|
|
| 65 |
return typed.length > 0 ? typed : null;
|
| 66 |
}
|
| 67 |
|
| 68 |
+
// ββ ElevenLabs voice βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 69 |
+
|
| 70 |
+
// Show the key box only until a key is stored; after that, show the confirmation
|
| 71 |
+
// and hide the field β the "onboard once, don't ask again" behaviour.
|
| 72 |
+
function reflectVoiceState(hasKey) {
|
| 73 |
+
els.voiceInputWrap.hidden = hasKey;
|
| 74 |
+
els.voiceEnabled.hidden = !hasKey;
|
| 75 |
+
if (hasKey) els.elevenKey.value = "";
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
async function loadVoiceConfig() {
|
| 79 |
+
try {
|
| 80 |
+
const resp = await fetch(VOICE_ENDPOINT);
|
| 81 |
+
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
| 82 |
+
const data = await resp.json();
|
| 83 |
+
reflectVoiceState(Boolean(data.has_api_key));
|
| 84 |
+
} catch (e) {
|
| 85 |
+
// A missing voice route (older build) just means no cloud voice β leave
|
| 86 |
+
// the box shown rather than blocking the page on it.
|
| 87 |
+
console.error("Failed to load voice config:", e);
|
| 88 |
+
}
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
// Save the ElevenLabs key only when the box is showing (onboarding or replace)
|
| 92 |
+
// and something was actually typed; otherwise there is nothing to do.
|
| 93 |
+
async function saveVoiceKey() {
|
| 94 |
+
if (els.voiceInputWrap.hidden) return;
|
| 95 |
+
const typed = els.elevenKey.value.trim();
|
| 96 |
+
if (!typed) return;
|
| 97 |
+
|
| 98 |
+
const resp = await fetch(VOICE_ENDPOINT, {
|
| 99 |
+
method: "POST",
|
| 100 |
+
headers: { "Content-Type": "application/json" },
|
| 101 |
+
body: JSON.stringify({ api_key: typed }),
|
| 102 |
+
});
|
| 103 |
+
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
|
| 104 |
+
const data = await resp.json();
|
| 105 |
+
reflectVoiceState(Boolean(data.has_api_key));
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
// ββ save βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 109 |
+
|
| 110 |
async function saveConfig(event) {
|
| 111 |
event.preventDefault();
|
| 112 |
|
|
|
|
| 130 |
reflectKeyState(Boolean(data.has_api_key));
|
| 131 |
els.apiKey.value = "";
|
| 132 |
els.clearKey.checked = false;
|
| 133 |
+
|
| 134 |
+
await saveVoiceKey();
|
| 135 |
+
|
| 136 |
setStatus("Saved. Restart ChittiOS to apply.", "ok");
|
| 137 |
} catch (e) {
|
| 138 |
setStatus("Could not save settings.", "err");
|
| 139 |
+
console.error("Failed to save settings:", e);
|
| 140 |
} finally {
|
| 141 |
els.saveBtn.disabled = false;
|
| 142 |
}
|
| 143 |
}
|
| 144 |
|
| 145 |
els.form.addEventListener("submit", saveConfig);
|
| 146 |
+
els.replaceVoice.addEventListener("click", () => {
|
| 147 |
+
reflectVoiceState(false);
|
| 148 |
+
els.elevenKey.focus();
|
| 149 |
+
});
|
| 150 |
+
|
| 151 |
loadConfig();
|
| 152 |
+
loadVoiceConfig();
|
chittios/static/style.css
CHANGED
|
@@ -200,6 +200,41 @@ input[type="password"]:focus {
|
|
| 200 |
color: var(--err);
|
| 201 |
}
|
| 202 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
.restart-note {
|
| 204 |
margin-top: 1.4rem;
|
| 205 |
padding-top: 1.1rem;
|
|
|
|
| 200 |
color: var(--err);
|
| 201 |
}
|
| 202 |
|
| 203 |
+
.voice {
|
| 204 |
+
margin-bottom: 1.4rem;
|
| 205 |
+
padding-top: 1.1rem;
|
| 206 |
+
border-top: 1px solid var(--line);
|
| 207 |
+
}
|
| 208 |
+
|
| 209 |
+
.voice .label {
|
| 210 |
+
margin-bottom: 0.6rem;
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
.voice-on {
|
| 214 |
+
display: flex;
|
| 215 |
+
align-items: baseline;
|
| 216 |
+
gap: 0.5rem;
|
| 217 |
+
flex-wrap: wrap;
|
| 218 |
+
font-size: 0.9rem;
|
| 219 |
+
color: var(--ok);
|
| 220 |
+
font-weight: 600;
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
.linkish {
|
| 224 |
+
font: inherit;
|
| 225 |
+
font-size: 0.85rem;
|
| 226 |
+
color: var(--muted);
|
| 227 |
+
background: none;
|
| 228 |
+
border: none;
|
| 229 |
+
padding: 0;
|
| 230 |
+
text-decoration: underline;
|
| 231 |
+
cursor: pointer;
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
.linkish:hover {
|
| 235 |
+
color: var(--accent);
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
.restart-note {
|
| 239 |
margin-top: 1.4rem;
|
| 240 |
padding-top: 1.1rem;
|
chittios_core/expression/config.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Where the ElevenLabs voice credentials come from β env, then file, then off.
|
| 2 |
+
|
| 3 |
+
The local Piper voice needs no configuration and always works. This resolves the
|
| 4 |
+
*optional* upgrade: an ElevenLabs key (and which voice to speak in), so a
|
| 5 |
+
household that wants the premium cloud voice provides a key once and every later
|
| 6 |
+
start picks it up. No key anywhere means ``enabled`` is false and the robot keeps
|
| 7 |
+
its local voice β the safe default, chosen by absence.
|
| 8 |
+
|
| 9 |
+
Precedence: env var > config file > default, per field
|
| 10 |
+
------------------------------------------------------
|
| 11 |
+
Mirrors the LLM config exactly, so the two behave the same to an operator:
|
| 12 |
+
|
| 13 |
+
1. an environment variable (``ELEVENLABS_API_KEY`` for the key β the name the
|
| 14 |
+
ElevenLabs ecosystem already uses β and ``CHITTIOS_TTS_VOICE_ID`` /
|
| 15 |
+
``CHITTIOS_TTS_MODEL`` for the rest);
|
| 16 |
+
2. the on-disk config file (``~/.config/chittios/voice.json``) the settings UI
|
| 17 |
+
writes when a household pastes a key at onboarding;
|
| 18 |
+
3. the built-in default β no key (so, off), a warm premade voice, the flash model.
|
| 19 |
+
|
| 20 |
+
The key is the one secret here, and it is treated like the LLM key: written to
|
| 21 |
+
the file, never echoed back over HTTP (the settings route reports only *whether*
|
| 22 |
+
one is set).
|
| 23 |
+
|
| 24 |
+
Why JSON, not TOML
|
| 25 |
+
------------------
|
| 26 |
+
The settings UI writes this file, and the standard library reads and writes JSON
|
| 27 |
+
losslessly with no third-party dependency β the same reasoning, and the same
|
| 28 |
+
install floor, as the LLM config beside it.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import json
|
| 34 |
+
import os
|
| 35 |
+
from collections.abc import Mapping
|
| 36 |
+
from dataclasses import dataclass
|
| 37 |
+
from pathlib import Path
|
| 38 |
+
from typing import Final
|
| 39 |
+
|
| 40 |
+
from chittios_core.expression.elevenlabs import (
|
| 41 |
+
DEFAULT_ELEVENLABS_MODEL,
|
| 42 |
+
DEFAULT_ELEVENLABS_VOICE_ID,
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
#: The config file the settings UI writes and this module reads. Beside the LLM
|
| 46 |
+
#: config under ``~/.config/chittios`` by the XDG convention.
|
| 47 |
+
DEFAULT_CONFIG_PATH: Final = Path.home() / ".config" / "chittios" / "voice.json"
|
| 48 |
+
|
| 49 |
+
#: Environment overrides, highest precedence. The key uses the ElevenLabs-standard
|
| 50 |
+
#: name so an operator who already exports it needs no ChittiOS-specific lookup.
|
| 51 |
+
ENV_API_KEY: Final = "ELEVENLABS_API_KEY"
|
| 52 |
+
ENV_VOICE_ID: Final = "CHITTIOS_TTS_VOICE_ID"
|
| 53 |
+
ENV_MODEL: Final = "CHITTIOS_TTS_MODEL"
|
| 54 |
+
|
| 55 |
+
#: The JSON object keys, shared by reader and writer so the two never drift.
|
| 56 |
+
_KEY_API_KEY: Final = "api_key"
|
| 57 |
+
_KEY_VOICE_ID: Final = "voice_id"
|
| 58 |
+
_KEY_MODEL: Final = "model_id"
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class VoiceConfigError(ValueError):
|
| 62 |
+
"""Raised when the on-disk voice config exists but cannot be used.
|
| 63 |
+
|
| 64 |
+
A present-but-malformed file is an operator error worth surfacing rather than
|
| 65 |
+
papering over: silently ignoring it would drop a household's chosen voice
|
| 66 |
+
without a word. Construction of the engine is guarded upstream, so this
|
| 67 |
+
degrades TTS to the local voice rather than stopping the robot from booting.
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@dataclass(frozen=True, slots=True)
|
| 72 |
+
class VoiceConfig:
|
| 73 |
+
"""The resolved cloud-voice settings: an optional key, a voice, a model.
|
| 74 |
+
|
| 75 |
+
Frozen because it is a startup snapshot, read once and handed to the engine
|
| 76 |
+
factory. A change to the file or the environment takes effect on the next app
|
| 77 |
+
start, never by mutating a live config.
|
| 78 |
+
|
| 79 |
+
Attributes:
|
| 80 |
+
api_key: The ElevenLabs key, or ``None`` when none is configured β the
|
| 81 |
+
single fact that decides whether the cloud voice is used at all.
|
| 82 |
+
voice_id: The voice to speak in.
|
| 83 |
+
model_id: The synthesis model to request.
|
| 84 |
+
"""
|
| 85 |
+
|
| 86 |
+
api_key: str | None
|
| 87 |
+
voice_id: str
|
| 88 |
+
model_id: str
|
| 89 |
+
|
| 90 |
+
@property
|
| 91 |
+
def enabled(self) -> bool:
|
| 92 |
+
"""Whether the cloud voice should be used β i.e. a key is configured."""
|
| 93 |
+
return self.api_key is not None
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def load_voice_config(
|
| 97 |
+
*,
|
| 98 |
+
path: Path = DEFAULT_CONFIG_PATH,
|
| 99 |
+
env: Mapping[str, str] | None = None,
|
| 100 |
+
) -> VoiceConfig:
|
| 101 |
+
"""Resolve the voice config: env var > file > default, per field.
|
| 102 |
+
|
| 103 |
+
Args:
|
| 104 |
+
path: The config file to read. A missing file is not an error β it means
|
| 105 |
+
"no key configured", and the cloud voice stays off.
|
| 106 |
+
env: The environment to read overrides from. Defaults to ``os.environ``;
|
| 107 |
+
injectable so a test can drive precedence without touching the
|
| 108 |
+
process environment.
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
The resolved :class:`VoiceConfig`.
|
| 112 |
+
|
| 113 |
+
Raises:
|
| 114 |
+
VoiceConfigError: if the file exists but is not a JSON object, or a value
|
| 115 |
+
is present with the wrong type.
|
| 116 |
+
"""
|
| 117 |
+
environ = os.environ if env is None else env
|
| 118 |
+
file_values = _read_config_file(path)
|
| 119 |
+
|
| 120 |
+
api_key = _resolve(ENV_API_KEY, _KEY_API_KEY, environ, file_values, None)
|
| 121 |
+
voice_id = (
|
| 122 |
+
_resolve(ENV_VOICE_ID, _KEY_VOICE_ID, environ, file_values, DEFAULT_ELEVENLABS_VOICE_ID)
|
| 123 |
+
or DEFAULT_ELEVENLABS_VOICE_ID
|
| 124 |
+
)
|
| 125 |
+
model_id = (
|
| 126 |
+
_resolve(ENV_MODEL, _KEY_MODEL, environ, file_values, DEFAULT_ELEVENLABS_MODEL)
|
| 127 |
+
or DEFAULT_ELEVENLABS_MODEL
|
| 128 |
+
)
|
| 129 |
+
return VoiceConfig(api_key=api_key, voice_id=voice_id, model_id=model_id)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def write_voice_config(config: VoiceConfig, *, path: Path = DEFAULT_CONFIG_PATH) -> None:
|
| 133 |
+
"""Write ``config`` to ``path`` as JSON, creating the directory if needed.
|
| 134 |
+
|
| 135 |
+
The settings UI's writer: it persists what a household pastes at onboarding so
|
| 136 |
+
the next start resolves it. ``api_key`` is written as JSON ``null`` when
|
| 137 |
+
unset, keeping the file a faithful, complete snapshot of all three settings.
|
| 138 |
+
|
| 139 |
+
Args:
|
| 140 |
+
config: The settings to persist.
|
| 141 |
+
path: Where to write them. The parent directory is created if absent.
|
| 142 |
+
"""
|
| 143 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 144 |
+
document = {
|
| 145 |
+
_KEY_API_KEY: config.api_key,
|
| 146 |
+
_KEY_VOICE_ID: config.voice_id,
|
| 147 |
+
_KEY_MODEL: config.model_id,
|
| 148 |
+
}
|
| 149 |
+
path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def _read_config_file(path: Path) -> Mapping[str, object]:
|
| 153 |
+
"""Read the config file into a mapping; an empty mapping if it does not exist.
|
| 154 |
+
|
| 155 |
+
Raises:
|
| 156 |
+
VoiceConfigError: if the file exists but does not hold a JSON object.
|
| 157 |
+
"""
|
| 158 |
+
if not path.exists():
|
| 159 |
+
return {}
|
| 160 |
+
try:
|
| 161 |
+
parsed: object = json.loads(path.read_text(encoding="utf-8"))
|
| 162 |
+
except (OSError, json.JSONDecodeError) as exc:
|
| 163 |
+
raise VoiceConfigError(f"could not read the voice config at {path}: {exc}") from exc
|
| 164 |
+
if not isinstance(parsed, dict):
|
| 165 |
+
raise VoiceConfigError(
|
| 166 |
+
f"the voice config at {path} must be a JSON object, got {type(parsed).__name__}"
|
| 167 |
+
)
|
| 168 |
+
return parsed
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _resolve(
|
| 172 |
+
env_key: str,
|
| 173 |
+
file_key: str,
|
| 174 |
+
env: Mapping[str, str],
|
| 175 |
+
file_values: Mapping[str, object],
|
| 176 |
+
default: str | None,
|
| 177 |
+
) -> str | None:
|
| 178 |
+
"""Return one field's value: env var > file entry > default.
|
| 179 |
+
|
| 180 |
+
A blank or whitespace-only environment variable counts as unset, so an
|
| 181 |
+
exported-but-empty ``ELEVENLABS_API_KEY`` falls through to the file rather
|
| 182 |
+
than pinning the key to the empty string. A file entry present as JSON
|
| 183 |
+
``null`` likewise means "unset" and falls through to the default.
|
| 184 |
+
"""
|
| 185 |
+
env_value = env.get(env_key)
|
| 186 |
+
if env_value is not None and env_value.strip():
|
| 187 |
+
return env_value
|
| 188 |
+
|
| 189 |
+
if file_key in file_values:
|
| 190 |
+
file_value = file_values[file_key]
|
| 191 |
+
if file_value is None:
|
| 192 |
+
return default
|
| 193 |
+
if not isinstance(file_value, str):
|
| 194 |
+
raise VoiceConfigError(
|
| 195 |
+
f"the voice config value for {file_key!r} must be a string or null, "
|
| 196 |
+
f"got {type(file_value).__name__}"
|
| 197 |
+
)
|
| 198 |
+
return file_value
|
| 199 |
+
|
| 200 |
+
return default
|
chittios_core/expression/elevenlabs.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ElevenLabs cloud text-to-speech β an optional, premium voice (CHT-TTS-cloud).
|
| 2 |
+
|
| 3 |
+
The local Piper voice is the default and the privacy floor: it never leaves the
|
| 4 |
+
box. This is the opposite trade a household can *opt into* β the robot's reply
|
| 5 |
+
text is sent to ElevenLabs and natural, expressive speech comes back β chosen by
|
| 6 |
+
pasting an API key at onboarding. It is deliberately not the default: a key must
|
| 7 |
+
be present for this engine to be built at all, and when it is absent, or fails,
|
| 8 |
+
the local voice speaks instead (see :class:`~chittios_core.expression.tts.FallbackTtsEngine`).
|
| 9 |
+
|
| 10 |
+
What crosses the wire, and what does not
|
| 11 |
+
----------------------------------------
|
| 12 |
+
Only the *reply* β text ChittiOS generated β goes to ElevenLabs. The family's
|
| 13 |
+
microphone audio never does; speech-to-text stays local by construction (that is
|
| 14 |
+
the line the project does not cross). Sending the robot's own words out for
|
| 15 |
+
synthesis is the same shape of trade the household already made when they pointed
|
| 16 |
+
conversation at a chosen LLM endpoint.
|
| 17 |
+
|
| 18 |
+
PCM straight onto the audio spine
|
| 19 |
+
---------------------------------
|
| 20 |
+
The request asks for ``pcm_16000`` β signed 16-bit little-endian mono at 16 kHz,
|
| 21 |
+
which *is* the rate the whole audio path standardises on. So the response decodes
|
| 22 |
+
with a single ``frombuffer`` and an int16βfloat32 scale: no MP3 decoder
|
| 23 |
+
dependency, and ``speak`` resamples nothing because the rate already matches.
|
| 24 |
+
|
| 25 |
+
Failures name themselves, and never mute the robot
|
| 26 |
+
--------------------------------------------------
|
| 27 |
+
A 4xx β a bad or unauthorised key, a plan that does not permit PCM, an exhausted
|
| 28 |
+
quota β is a standing condition, raised as :class:`TtsUnavailableError` so the
|
| 29 |
+
fallback engine latches to the local voice for the session. A 5xx or a network
|
| 30 |
+
blip is transient, raised as :class:`TtsTransientError` so the fallback covers
|
| 31 |
+
just that one line and the next line tries the cloud again.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
+
from typing import Final
|
| 37 |
+
|
| 38 |
+
import httpx
|
| 39 |
+
import numpy as np
|
| 40 |
+
from numpy.typing import NDArray
|
| 41 |
+
|
| 42 |
+
from chittios_core.expression.tts import TtsTransientError, TtsUnavailableError
|
| 43 |
+
from hal.reference_hal import AUDIO_DTYPE
|
| 44 |
+
|
| 45 |
+
#: ElevenLabs' low-latency model β fast and inexpensive per character, the right
|
| 46 |
+
#: default for a robot that speaks in short conversational turns rather than
|
| 47 |
+
#: narrating essays. Overridable per household via config.
|
| 48 |
+
DEFAULT_ELEVENLABS_MODEL: Final = "eleven_flash_v2_5"
|
| 49 |
+
|
| 50 |
+
#: "George" β a warm, premade storyteller voice. Premade voice ids are shared
|
| 51 |
+
#: across every ElevenLabs account, so this default resolves for any customer's
|
| 52 |
+
#: key, not just the developer's. Overridable per household via config.
|
| 53 |
+
DEFAULT_ELEVENLABS_VOICE_ID: Final = "JBFqnCBsd6RMkjVDRZzb"
|
| 54 |
+
|
| 55 |
+
#: The API host. A constant, not a config field: unlike the LLM endpoint there is
|
| 56 |
+
#: one ElevenLabs, and a household chooses the voice, not the server.
|
| 57 |
+
DEFAULT_ELEVENLABS_BASE_URL: Final = "https://api.elevenlabs.io"
|
| 58 |
+
|
| 59 |
+
#: The one output format requested, and the rate it carries. Both are fixed
|
| 60 |
+
#: together: ``pcm_16000`` is signed 16-bit LE mono at exactly this rate, so the
|
| 61 |
+
#: decode is a ``frombuffer`` and no resample is ever needed downstream.
|
| 62 |
+
_OUTPUT_FORMAT: Final = "pcm_16000"
|
| 63 |
+
_SAMPLE_RATE_HZ: Final = 16_000
|
| 64 |
+
|
| 65 |
+
#: How long to wait on one synthesis before calling it a transient failure. Long
|
| 66 |
+
#: enough for a sentence on a slow link, short enough that a hung request falls
|
| 67 |
+
#: back to the local voice rather than stalling the turn.
|
| 68 |
+
_DEFAULT_TIMEOUT_S: Final = 30.0
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class ElevenLabsTtsEngine:
|
| 72 |
+
"""Synthesizes text through the ElevenLabs cloud API, returning 16 kHz mono.
|
| 73 |
+
|
| 74 |
+
Satisfies the :class:`~chittios_core.expression.tts.TtsEngine` protocol, so it
|
| 75 |
+
drops into ``speak`` exactly where Piper or the stub would. Build it only when
|
| 76 |
+
a key is configured; wrap it in a ``FallbackTtsEngine`` so a cloud failure is
|
| 77 |
+
covered by the local voice rather than heard as silence.
|
| 78 |
+
|
| 79 |
+
Args:
|
| 80 |
+
api_key: The household's ElevenLabs API key. Required and non-empty.
|
| 81 |
+
voice_id: The voice to speak in. Defaults to a warm premade voice.
|
| 82 |
+
model_id: The synthesis model. Defaults to the low-latency flash model.
|
| 83 |
+
base_url: The API host. Defaults to ElevenLabs.
|
| 84 |
+
timeout_s: Per-request timeout in seconds.
|
| 85 |
+
client: An injected HTTP client, for tests. Defaults to a pooled
|
| 86 |
+
:class:`httpx.Client`, so successive turns reuse one TLS connection
|
| 87 |
+
rather than paying a fresh handshake each time.
|
| 88 |
+
|
| 89 |
+
Raises:
|
| 90 |
+
TtsUnavailableError: if ``api_key`` is empty β the one failure knowable at
|
| 91 |
+
construction, kept here so a blank key never yields a live-but-useless
|
| 92 |
+
engine.
|
| 93 |
+
"""
|
| 94 |
+
|
| 95 |
+
def __init__(
|
| 96 |
+
self,
|
| 97 |
+
api_key: str,
|
| 98 |
+
*,
|
| 99 |
+
voice_id: str = DEFAULT_ELEVENLABS_VOICE_ID,
|
| 100 |
+
model_id: str = DEFAULT_ELEVENLABS_MODEL,
|
| 101 |
+
base_url: str = DEFAULT_ELEVENLABS_BASE_URL,
|
| 102 |
+
timeout_s: float = _DEFAULT_TIMEOUT_S,
|
| 103 |
+
client: httpx.Client | None = None,
|
| 104 |
+
) -> None:
|
| 105 |
+
if not api_key or not api_key.strip():
|
| 106 |
+
raise TtsUnavailableError("ElevenLabsTtsEngine needs a non-empty API key.")
|
| 107 |
+
self._api_key = api_key
|
| 108 |
+
self._model_id = model_id
|
| 109 |
+
self._url = f"{base_url.rstrip('/')}/v1/text-to-speech/{voice_id}"
|
| 110 |
+
self._client = client if client is not None else httpx.Client(timeout=timeout_s)
|
| 111 |
+
|
| 112 |
+
def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
|
| 113 |
+
"""Synthesize ``text`` and return ``(mono float32 samples, 16000)``.
|
| 114 |
+
|
| 115 |
+
Whitespace-only or empty text short-circuits to empty audio β there is
|
| 116 |
+
nothing to say and no reason to spend a request or a character of quota.
|
| 117 |
+
|
| 118 |
+
Raises:
|
| 119 |
+
TtsUnavailableError: on a 4xx β a standing condition (bad key, a plan
|
| 120 |
+
without PCM, exhausted quota) the caller should stop retrying.
|
| 121 |
+
TtsTransientError: on a 5xx or a network error β worth trying again on
|
| 122 |
+
the next line.
|
| 123 |
+
"""
|
| 124 |
+
if not text.strip():
|
| 125 |
+
return np.zeros(0, dtype=AUDIO_DTYPE), _SAMPLE_RATE_HZ
|
| 126 |
+
|
| 127 |
+
try:
|
| 128 |
+
response = self._client.post(
|
| 129 |
+
self._url,
|
| 130 |
+
params={"output_format": _OUTPUT_FORMAT},
|
| 131 |
+
headers={"xi-api-key": self._api_key, "content-type": "application/json"},
|
| 132 |
+
json={"text": text, "model_id": self._model_id},
|
| 133 |
+
)
|
| 134 |
+
except httpx.HTTPError as exc:
|
| 135 |
+
raise TtsTransientError(f"could not reach ElevenLabs: {exc}") from exc
|
| 136 |
+
|
| 137 |
+
if response.status_code != 200:
|
| 138 |
+
detail = _short_detail(response)
|
| 139 |
+
if 400 <= response.status_code < 500:
|
| 140 |
+
raise TtsUnavailableError(
|
| 141 |
+
f"ElevenLabs rejected the request (HTTP {response.status_code}): {detail}"
|
| 142 |
+
)
|
| 143 |
+
raise TtsTransientError(
|
| 144 |
+
f"ElevenLabs is unavailable (HTTP {response.status_code}): {detail}"
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
pcm = np.frombuffer(response.content, dtype="<i2")
|
| 148 |
+
# int16 β float32 in [-1, 1). Dividing by 32768 (not 32767) keeps a
|
| 149 |
+
# full-scale negative sample exactly at -1.0, matching the convention the
|
| 150 |
+
# rest of the audio spine uses.
|
| 151 |
+
samples = (pcm.astype(np.float32) / 32768.0).astype(AUDIO_DTYPE)
|
| 152 |
+
return samples, _SAMPLE_RATE_HZ
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def _short_detail(response: httpx.Response) -> str:
|
| 156 |
+
"""A brief, safe description of a failed response for a log line.
|
| 157 |
+
|
| 158 |
+
The body is truncated because ElevenLabs' error JSON can be long, and this
|
| 159 |
+
lands in a warning the household may read β the status and a hint, not a wall.
|
| 160 |
+
"""
|
| 161 |
+
try:
|
| 162 |
+
return response.text[:200]
|
| 163 |
+
except Exception: # pragma: no cover - a body that will not decode is still a failure
|
| 164 |
+
return "<unreadable response body>"
|
chittios_core/expression/tts.py
CHANGED
|
@@ -1,305 +1,373 @@
|
|
| 1 |
-
"""Local text-to-speech for the expression subsystem (CHT-019, Phase 2).
|
| 2 |
-
|
| 3 |
-
Layer 2's voice. Core turns a line of text into audio and streams it to the
|
| 4 |
-
body through the HAL contract's ``audio_out``. Everything here runs on the
|
| 5 |
-
machine β no cloud, no network β which is the whole promise of the project:
|
| 6 |
-
*"With the local setup, audio never leaves your machine."* (spec Β§9).
|
| 7 |
-
|
| 8 |
-
Three deliberate shapes
|
| 9 |
-
-----------------------
|
| 10 |
-
**A ``TtsEngine`` seam.** Synthesis is the one part that wants a heavy,
|
| 11 |
-
optional runtime (an ONNX voice model). Putting it behind a Protocol keeps the
|
| 12 |
-
streaming, resampling, and chunking logic β the part Core actually depends on β
|
| 13 |
-
testable without that runtime, the same way ``identity.voice`` keeps its
|
| 14 |
-
matching policy testable without torch. A ``StubTtsEngine`` satisfies the seam
|
| 15 |
-
with deterministic synthetic audio; ``PiperTtsEngine`` is the real backend and
|
| 16 |
-
is imported lazily so this module stays importable on a machine with no Piper.
|
| 17 |
-
|
| 18 |
-
**Everything lands at 16 kHz float32 mono.** The Reachy Mini push path
|
| 19 |
-
(``media.push_audio_sample``) wants float32 at 16 kHz, and the HAL contract
|
| 20 |
-
fixes float32 as the processing format end to end (``AUDIO_DTYPE``). A voice
|
| 21 |
-
model that speaks at 22.05 kHz is normal, so ``speak`` resamples whatever the
|
| 22 |
-
engine produces onto the target rate rather than trusting engines to agree.
|
| 23 |
-
Resampling once, here, means no downstream stage has to.
|
| 24 |
-
|
| 25 |
-
**Chunked, not buffered.** ``speak`` emits ~100 ms ``AudioChunk``s as it goes
|
| 26 |
-
so the body can begin speaking before the whole utterance is synthesized and
|
| 27 |
-
resampled. That streaming is what the narration-latency budget in spec Β§7.2
|
| 28 |
-
(time-to-first-audio) is built on; buffering the full clip first would spend
|
| 29 |
-
the entire budget before the first sample reached the speaker.
|
| 30 |
-
"""
|
| 31 |
-
|
| 32 |
-
from __future__ import annotations
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
from
|
| 37 |
-
|
| 38 |
-
import
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
from
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
#:
|
| 49 |
-
#:
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
#:
|
| 54 |
-
#:
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
"
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
self
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
if
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
"""
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
from
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
"""
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Local text-to-speech for the expression subsystem (CHT-019, Phase 2).
|
| 2 |
+
|
| 3 |
+
Layer 2's voice. Core turns a line of text into audio and streams it to the
|
| 4 |
+
body through the HAL contract's ``audio_out``. Everything here runs on the
|
| 5 |
+
machine β no cloud, no network β which is the whole promise of the project:
|
| 6 |
+
*"With the local setup, audio never leaves your machine."* (spec Β§9).
|
| 7 |
+
|
| 8 |
+
Three deliberate shapes
|
| 9 |
+
-----------------------
|
| 10 |
+
**A ``TtsEngine`` seam.** Synthesis is the one part that wants a heavy,
|
| 11 |
+
optional runtime (an ONNX voice model). Putting it behind a Protocol keeps the
|
| 12 |
+
streaming, resampling, and chunking logic β the part Core actually depends on β
|
| 13 |
+
testable without that runtime, the same way ``identity.voice`` keeps its
|
| 14 |
+
matching policy testable without torch. A ``StubTtsEngine`` satisfies the seam
|
| 15 |
+
with deterministic synthetic audio; ``PiperTtsEngine`` is the real backend and
|
| 16 |
+
is imported lazily so this module stays importable on a machine with no Piper.
|
| 17 |
+
|
| 18 |
+
**Everything lands at 16 kHz float32 mono.** The Reachy Mini push path
|
| 19 |
+
(``media.push_audio_sample``) wants float32 at 16 kHz, and the HAL contract
|
| 20 |
+
fixes float32 as the processing format end to end (``AUDIO_DTYPE``). A voice
|
| 21 |
+
model that speaks at 22.05 kHz is normal, so ``speak`` resamples whatever the
|
| 22 |
+
engine produces onto the target rate rather than trusting engines to agree.
|
| 23 |
+
Resampling once, here, means no downstream stage has to.
|
| 24 |
+
|
| 25 |
+
**Chunked, not buffered.** ``speak`` emits ~100 ms ``AudioChunk``s as it goes
|
| 26 |
+
so the body can begin speaking before the whole utterance is synthesized and
|
| 27 |
+
resampled. That streaming is what the narration-latency budget in spec Β§7.2
|
| 28 |
+
(time-to-first-audio) is built on; buffering the full clip first would spend
|
| 29 |
+
the entire budget before the first sample reached the speaker.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
from __future__ import annotations
|
| 33 |
+
|
| 34 |
+
import asyncio
|
| 35 |
+
import logging
|
| 36 |
+
from collections.abc import AsyncIterator
|
| 37 |
+
from math import gcd
|
| 38 |
+
from typing import Final, Protocol, runtime_checkable
|
| 39 |
+
|
| 40 |
+
import numpy as np
|
| 41 |
+
from numpy.typing import NDArray
|
| 42 |
+
|
| 43 |
+
from hal.reference_hal import AUDIO_DTYPE, AudioChunk
|
| 44 |
+
|
| 45 |
+
logger = logging.getLogger(__name__)
|
| 46 |
+
|
| 47 |
+
#: The rate every utterance is delivered at. The Reachy Mini push path expects
|
| 48 |
+
#: float32 @ 16 kHz, and 16 kHz is already the rate the perception and voice
|
| 49 |
+
#: paths standardize on, so one constant governs the whole audio spine.
|
| 50 |
+
TARGET_SAMPLE_RATE_HZ: Final = 16_000
|
| 51 |
+
|
| 52 |
+
#: Chunk granularity for streaming playback. ~100 ms is small enough that
|
| 53 |
+
#: time-to-first-audio stays inside the budget yet large enough that the
|
| 54 |
+
#: per-chunk overhead (an ``AudioChunk`` allocation and an ``await``) is noise.
|
| 55 |
+
DEFAULT_CHUNK_MS: Final = 100
|
| 56 |
+
|
| 57 |
+
#: Piper's default voices speak at 22.05 kHz. Used by ``StubTtsEngine`` so the
|
| 58 |
+
#: default test path exercises the resampler rather than the pass-through.
|
| 59 |
+
DEFAULT_STUB_SAMPLE_RATE_HZ: Final = 22_050
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class TtsUnavailableError(RuntimeError):
|
| 63 |
+
"""Raised when a TTS backend cannot be used, and the caller should stop trying.
|
| 64 |
+
|
| 65 |
+
Two shapes of standing failure: a backend's runtime dependency is not
|
| 66 |
+
installed (surfaced at construction, naming the extra to install), or a cloud
|
| 67 |
+
backend was refused for a durable reason β a bad key, a plan without the
|
| 68 |
+
requested format, an exhausted quota (a 4xx). Both mean "do not keep retrying
|
| 69 |
+
this backend"; :class:`FallbackTtsEngine` latches to its fallback on this.
|
| 70 |
+
"""
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class TtsTransientError(RuntimeError):
|
| 74 |
+
"""Raised when a TTS backend failed in a way that may succeed next time.
|
| 75 |
+
|
| 76 |
+
A network blip or a 5xx from a cloud backend β worth covering with the
|
| 77 |
+
fallback for just this one line, then trying the primary again on the next.
|
| 78 |
+
The counterpart to :class:`TtsUnavailableError`, which is durable.
|
| 79 |
+
"""
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
@runtime_checkable
|
| 83 |
+
class TtsEngine(Protocol):
|
| 84 |
+
"""Turns text into a single block of mono audio at the engine's own rate.
|
| 85 |
+
|
| 86 |
+
The engine owns synthesis and nothing else: it does not resample, chunk, or
|
| 87 |
+
touch the body. Returning the rate alongside the samples β rather than
|
| 88 |
+
assuming a fixed one β is what lets ``speak`` accept any voice model and
|
| 89 |
+
still guarantee 16 kHz downstream.
|
| 90 |
+
"""
|
| 91 |
+
|
| 92 |
+
def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
|
| 93 |
+
"""Return ``(mono float32 samples in [-1, 1], sample_rate_hz)``."""
|
| 94 |
+
...
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class StubTtsEngine:
|
| 98 |
+
"""Deterministic synthetic speech for tests β no external dependency.
|
| 99 |
+
|
| 100 |
+
Emits a fixed-frequency tone whose length grows with the text, so callers
|
| 101 |
+
get non-empty, chunk-able audio without a voice model. Deterministic by
|
| 102 |
+
construction: the same text always yields identical samples, which is what
|
| 103 |
+
lets tests assert on exact behaviour instead of tolerating model jitter.
|
| 104 |
+
|
| 105 |
+
Args:
|
| 106 |
+
sample_rate_hz: Rate of the produced audio. Defaults to Piper's
|
| 107 |
+
22.05 kHz so the default path deliberately differs from the 16 kHz
|
| 108 |
+
target and exercises the resampler.
|
| 109 |
+
tone_hz: Frequency of the synthesized tone.
|
| 110 |
+
seconds_per_char: Audio duration contributed by each character.
|
| 111 |
+
amplitude: Peak amplitude, comfortably inside [-1, 1].
|
| 112 |
+
"""
|
| 113 |
+
|
| 114 |
+
def __init__(
|
| 115 |
+
self,
|
| 116 |
+
*,
|
| 117 |
+
sample_rate_hz: int = DEFAULT_STUB_SAMPLE_RATE_HZ,
|
| 118 |
+
tone_hz: float = 220.0,
|
| 119 |
+
seconds_per_char: float = 0.08,
|
| 120 |
+
amplitude: float = 0.25,
|
| 121 |
+
) -> None:
|
| 122 |
+
if sample_rate_hz <= 0:
|
| 123 |
+
raise ValueError(f"sample_rate_hz must be positive, got {sample_rate_hz}")
|
| 124 |
+
if tone_hz <= 0:
|
| 125 |
+
raise ValueError(f"tone_hz must be positive, got {tone_hz}")
|
| 126 |
+
if seconds_per_char <= 0:
|
| 127 |
+
raise ValueError(f"seconds_per_char must be positive, got {seconds_per_char}")
|
| 128 |
+
if not 0.0 < amplitude <= 1.0:
|
| 129 |
+
raise ValueError(f"amplitude must lie in (0, 1], got {amplitude}")
|
| 130 |
+
self._sample_rate_hz = sample_rate_hz
|
| 131 |
+
self._tone_hz = tone_hz
|
| 132 |
+
self._seconds_per_char = seconds_per_char
|
| 133 |
+
self._amplitude = amplitude
|
| 134 |
+
|
| 135 |
+
def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
|
| 136 |
+
"""Return a tone whose duration scales with ``len(text)``.
|
| 137 |
+
|
| 138 |
+
Whitespace-only or empty text yields an empty array β the honest
|
| 139 |
+
representation of "nothing to say" β which ``speak`` streams as zero
|
| 140 |
+
chunks rather than a silent blip.
|
| 141 |
+
"""
|
| 142 |
+
char_count = len(text.strip())
|
| 143 |
+
if char_count == 0:
|
| 144 |
+
return np.zeros(0, dtype=AUDIO_DTYPE), self._sample_rate_hz
|
| 145 |
+
|
| 146 |
+
duration_s = char_count * self._seconds_per_char
|
| 147 |
+
sample_count = int(self._sample_rate_hz * duration_s)
|
| 148 |
+
t = np.arange(sample_count, dtype=np.float64) / self._sample_rate_hz
|
| 149 |
+
samples = self._amplitude * np.sin(2.0 * np.pi * self._tone_hz * t)
|
| 150 |
+
return samples.astype(AUDIO_DTYPE), self._sample_rate_hz
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class PiperTtsEngine:
|
| 154 |
+
"""Piper-backed synthesis, behind the optional ``voice-tts`` extra.
|
| 155 |
+
|
| 156 |
+
Piper is a local, ONNX-based neural TTS: no cloud, fast on CPU, which is
|
| 157 |
+
exactly the constraint a household robot's voice lives under. It is an
|
| 158 |
+
optional dependency β importing this module never requires it β so the
|
| 159 |
+
import is deferred to construction and a clear ``TtsUnavailableError`` is
|
| 160 |
+
raised when it is absent, naming the extra to install.
|
| 161 |
+
|
| 162 |
+
Args:
|
| 163 |
+
model_path: Path to a Piper ``.onnx`` voice model.
|
| 164 |
+
config_path: Path to the model's ``.onnx.json`` config. Piper defaults
|
| 165 |
+
to ``<model_path>.json`` when omitted, so this is only needed when
|
| 166 |
+
the config sits elsewhere.
|
| 167 |
+
use_cuda: Run inference on GPU. Off by default β the target is a CPU
|
| 168 |
+
SBC, and CUDA is the exception, not the rule.
|
| 169 |
+
|
| 170 |
+
Raises:
|
| 171 |
+
TtsUnavailableError: if ``piper-tts`` is not installed.
|
| 172 |
+
"""
|
| 173 |
+
|
| 174 |
+
def __init__(
|
| 175 |
+
self,
|
| 176 |
+
model_path: str,
|
| 177 |
+
*,
|
| 178 |
+
config_path: str | None = None,
|
| 179 |
+
use_cuda: bool = False,
|
| 180 |
+
) -> None:
|
| 181 |
+
try:
|
| 182 |
+
from piper import PiperVoice
|
| 183 |
+
except ImportError as exc: # pragma: no cover - exercised only without piper
|
| 184 |
+
raise TtsUnavailableError(
|
| 185 |
+
"PiperTtsEngine needs the 'piper-tts' package, which is not installed. "
|
| 186 |
+
"Install the optional extra: pip install 'chittios[voice-tts]'."
|
| 187 |
+
) from exc
|
| 188 |
+
|
| 189 |
+
self._voice = PiperVoice.load(model_path, config_path=config_path, use_cuda=use_cuda)
|
| 190 |
+
|
| 191 |
+
def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
|
| 192 |
+
"""Synthesize ``text`` and return ``(mono float32 samples, rate)``.
|
| 193 |
+
|
| 194 |
+
Piper streams int16 chunks; they are concatenated and converted to the
|
| 195 |
+
float32 the rest of the pipeline speaks in. ``speak`` re-chunks the
|
| 196 |
+
result for playback, so gathering Piper's chunks here keeps this method
|
| 197 |
+
a plain ``TtsEngine`` β text in, one waveform out β and leaves streaming
|
| 198 |
+
policy in one place.
|
| 199 |
+
|
| 200 |
+
Raises:
|
| 201 |
+
TtsUnavailableError: if Piper yields no audio for the text.
|
| 202 |
+
"""
|
| 203 |
+
blocks: list[NDArray[np.int16]] = []
|
| 204 |
+
sample_rate: int | None = None
|
| 205 |
+
for chunk in self._voice.synthesize(text):
|
| 206 |
+
sample_rate = chunk.sample_rate
|
| 207 |
+
blocks.append(np.frombuffer(chunk.audio_int16_bytes, dtype=np.int16))
|
| 208 |
+
|
| 209 |
+
if sample_rate is None:
|
| 210 |
+
raise TtsUnavailableError(f"Piper produced no audio for text: {text!r}")
|
| 211 |
+
|
| 212 |
+
if blocks:
|
| 213 |
+
pcm = np.concatenate(blocks)
|
| 214 |
+
else:
|
| 215 |
+
pcm = np.zeros(0, dtype=np.int16)
|
| 216 |
+
|
| 217 |
+
# int16 -> float32 in [-1, 1). Dividing by 32768 (not 32767) is the
|
| 218 |
+
# standard convention: it keeps full-scale negative samples exactly at
|
| 219 |
+
# -1.0 and never overshoots the range the contract promises.
|
| 220 |
+
samples = (pcm.astype(np.float32) / 32768.0).astype(AUDIO_DTYPE)
|
| 221 |
+
return samples, sample_rate
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
class FallbackTtsEngine:
|
| 225 |
+
"""Speaks through a primary engine, falling back to a local one on failure.
|
| 226 |
+
|
| 227 |
+
The one place the cloud/local trade is arbitrated: a household that has opted
|
| 228 |
+
into ElevenLabs gets it, but a bad key, an exhausted quota, or a dropped
|
| 229 |
+
connection is never heard as silence β the local voice covers the line
|
| 230 |
+
instead. This keeps every other layer unaware there is a choice at all; it
|
| 231 |
+
still holds one ``TtsEngine`` and calls ``synthesize``.
|
| 232 |
+
|
| 233 |
+
Durable vs transient, from the raised error:
|
| 234 |
+
|
| 235 |
+
- :class:`TtsUnavailableError` is a standing condition (bad key, plan without
|
| 236 |
+
PCM, quota gone). The primary is *disabled for the session* β retrying it
|
| 237 |
+
every sentence would add a doomed round-trip to each turn β and a restart is
|
| 238 |
+
what re-checks it, once the household has fixed the cause.
|
| 239 |
+
- :class:`TtsTransientError` (or any unexpected error) covers just this line;
|
| 240 |
+
the next line tries the primary again.
|
| 241 |
+
|
| 242 |
+
Args:
|
| 243 |
+
primary: The preferred engine (e.g. ElevenLabs).
|
| 244 |
+
fallback: The always-available local engine (Piper, or the stub tone).
|
| 245 |
+
"""
|
| 246 |
+
|
| 247 |
+
def __init__(self, primary: TtsEngine, fallback: TtsEngine) -> None:
|
| 248 |
+
self._primary = primary
|
| 249 |
+
self._fallback = fallback
|
| 250 |
+
self._primary_disabled = False
|
| 251 |
+
|
| 252 |
+
def synthesize(self, text: str) -> tuple[NDArray[np.float32], int]:
|
| 253 |
+
"""Synthesize with the primary engine, or the fallback if it cannot."""
|
| 254 |
+
if not self._primary_disabled:
|
| 255 |
+
try:
|
| 256 |
+
return self._primary.synthesize(text)
|
| 257 |
+
except TtsUnavailableError as exc:
|
| 258 |
+
logger.warning(
|
| 259 |
+
"primary voice unavailable (%s); using the local voice for the rest "
|
| 260 |
+
"of this session β restart once the cause is fixed to re-enable it",
|
| 261 |
+
exc,
|
| 262 |
+
)
|
| 263 |
+
self._primary_disabled = True
|
| 264 |
+
except TtsTransientError as exc:
|
| 265 |
+
logger.warning(
|
| 266 |
+
"primary voice hiccup (%s); using the local voice for this line", exc
|
| 267 |
+
)
|
| 268 |
+
except Exception:
|
| 269 |
+
logger.exception("primary voice raised unexpectedly; using the local voice")
|
| 270 |
+
return self._fallback.synthesize(text)
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _resample_to_target(samples: NDArray[np.float32], source_rate_hz: int) -> NDArray[np.float32]:
|
| 274 |
+
"""Resample mono audio onto ``TARGET_SAMPLE_RATE_HZ`` float32.
|
| 275 |
+
|
| 276 |
+
A pass-through when the rates already match, so the common Piper-at-16 kHz
|
| 277 |
+
case pays nothing. Otherwise polyphase resampling via ``scipy.signal`` β
|
| 278 |
+
imported lazily, matching the HAL adapters, so this module stays importable
|
| 279 |
+
without SciPy installed. Reducing up/down by their GCD keeps the polyphase
|
| 280 |
+
filter small (16 kHz from 22.05 kHz is 320/441, not 16000/22050).
|
| 281 |
+
|
| 282 |
+
Args:
|
| 283 |
+
samples: Mono float32 at ``source_rate_hz``.
|
| 284 |
+
source_rate_hz: The rate ``samples`` were produced at.
|
| 285 |
+
|
| 286 |
+
Returns:
|
| 287 |
+
Contiguous float32 at ``TARGET_SAMPLE_RATE_HZ``, clipped to [-1, 1]
|
| 288 |
+
because polyphase filtering can ring a hair past full scale.
|
| 289 |
+
"""
|
| 290 |
+
if samples.ndim != 1:
|
| 291 |
+
raise ValueError(f"expected mono (1-D) audio, got shape {samples.shape}")
|
| 292 |
+
if source_rate_hz <= 0:
|
| 293 |
+
raise ValueError(f"source_rate_hz must be positive, got {source_rate_hz}")
|
| 294 |
+
if source_rate_hz == TARGET_SAMPLE_RATE_HZ or samples.size == 0:
|
| 295 |
+
return np.ascontiguousarray(samples, dtype=AUDIO_DTYPE)
|
| 296 |
+
|
| 297 |
+
from scipy.signal import resample_poly
|
| 298 |
+
|
| 299 |
+
divisor = gcd(source_rate_hz, TARGET_SAMPLE_RATE_HZ)
|
| 300 |
+
up = TARGET_SAMPLE_RATE_HZ // divisor
|
| 301 |
+
down = source_rate_hz // divisor
|
| 302 |
+
resampled = resample_poly(samples.astype(np.float64), up, down)
|
| 303 |
+
return np.ascontiguousarray(np.clip(resampled, -1.0, 1.0), dtype=AUDIO_DTYPE)
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
async def _stream_chunks(
|
| 307 |
+
samples: NDArray[np.float32], *, chunk_ms: int
|
| 308 |
+
) -> AsyncIterator[AudioChunk]:
|
| 309 |
+
"""Slice 16 kHz mono ``samples`` into ~``chunk_ms`` ``AudioChunk``s.
|
| 310 |
+
|
| 311 |
+
Timestamps advance by each chunk's own duration so the stream carries a
|
| 312 |
+
coherent playback clock. A short final chunk is yielded as-is rather than
|
| 313 |
+
padded β padding would inject silence the speaker would actually play.
|
| 314 |
+
"""
|
| 315 |
+
samples_per_chunk = int(TARGET_SAMPLE_RATE_HZ * chunk_ms / 1000)
|
| 316 |
+
elapsed_s = 0.0
|
| 317 |
+
for start in range(0, len(samples), samples_per_chunk):
|
| 318 |
+
block = np.ascontiguousarray(samples[start : start + samples_per_chunk], dtype=AUDIO_DTYPE)
|
| 319 |
+
yield AudioChunk(
|
| 320 |
+
samples=block,
|
| 321 |
+
sample_rate_hz=TARGET_SAMPLE_RATE_HZ,
|
| 322 |
+
channels=1,
|
| 323 |
+
timestamp_s=elapsed_s,
|
| 324 |
+
)
|
| 325 |
+
elapsed_s += len(block) / TARGET_SAMPLE_RATE_HZ
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
@runtime_checkable
|
| 329 |
+
class SupportsAudioOut(Protocol):
|
| 330 |
+
"""The slice of a HAL body that ``speak`` needs: a sink for an audio stream.
|
| 331 |
+
|
| 332 |
+
Typing ``speak`` to this narrow shape (not the full ``HALAdapter``) is what
|
| 333 |
+
lets the screen-avatar test double β which implements only ``audio_out`` β
|
| 334 |
+
stand in for the real Reachy body, and keeps ``speak`` honest about what it
|
| 335 |
+
actually touches.
|
| 336 |
+
"""
|
| 337 |
+
|
| 338 |
+
async def audio_out(self, chunks: AsyncIterator[AudioChunk]) -> None:
|
| 339 |
+
"""Play a stream of audio chunks, returning once they are consumed."""
|
| 340 |
+
...
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
async def speak(
|
| 344 |
+
text: str,
|
| 345 |
+
body: SupportsAudioOut,
|
| 346 |
+
engine: TtsEngine,
|
| 347 |
+
*,
|
| 348 |
+
chunk_ms: int = DEFAULT_CHUNK_MS,
|
| 349 |
+
) -> None:
|
| 350 |
+
"""Synthesize ``text`` and stream it to ``body`` as 16 kHz float32 mono.
|
| 351 |
+
|
| 352 |
+
The full expression path for one line: synthesize with ``engine``, resample
|
| 353 |
+
to the body's expected rate, and stream ~``chunk_ms`` chunks into
|
| 354 |
+
``body.audio_out`` so playback can start before synthesis finishes. Empty
|
| 355 |
+
or whitespace-only text streams nothing β there is no audio to play, and an
|
| 356 |
+
empty stream is the correct representation of silence.
|
| 357 |
+
|
| 358 |
+
Args:
|
| 359 |
+
text: The line to speak.
|
| 360 |
+
body: The HAL adapter that will play the audio.
|
| 361 |
+
engine: The synthesis backend.
|
| 362 |
+
chunk_ms: Playback chunk duration. Must be positive.
|
| 363 |
+
"""
|
| 364 |
+
if chunk_ms <= 0:
|
| 365 |
+
raise ValueError(f"chunk_ms must be positive, got {chunk_ms}")
|
| 366 |
+
|
| 367 |
+
# Synthesis is a blocking call β a Piper decode on a slow CPU, or a network
|
| 368 |
+
# round-trip to a cloud engine β so it runs on a worker thread. On the event
|
| 369 |
+
# loop it would freeze everything (a still-speaking reply, motion, the stop
|
| 370 |
+
# check) for its whole duration; off it, the loop keeps serving them.
|
| 371 |
+
raw_samples, source_rate_hz = await asyncio.to_thread(engine.synthesize, text)
|
| 372 |
+
samples = _resample_to_target(raw_samples, source_rate_hz)
|
| 373 |
+
await body.audio_out(_stream_chunks(samples, chunk_ms=chunk_ms))
|
pyproject.toml
CHANGED
|
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
|
|
| 5 |
|
| 6 |
[project]
|
| 7 |
name = "chittios"
|
| 8 |
-
version = "1.
|
| 9 |
description = "ChittiOS β a local-first household companion for Reachy Mini"
|
| 10 |
readme = "README.md"
|
| 11 |
requires-python = ">=3.10"
|
|
|
|
| 5 |
|
| 6 |
[project]
|
| 7 |
name = "chittios"
|
| 8 |
+
version = "1.2.0"
|
| 9 |
description = "ChittiOS β a local-first household companion for Reachy Mini"
|
| 10 |
readme = "README.md"
|
| 11 |
requires-python = ">=3.10"
|