diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..000cfc642505e33095584699dd0448183489e600 --- /dev/null +++ b/.env.example @@ -0,0 +1,88 @@ +OPENAI_API_KEY= +MODEL_NAME="gpt-realtime-2" + +# Local vision model (only used with --local-vision CLI flag) +# By default, vision is handled by gpt-realtime when the camera tool is used +LOCAL_VISION_MODEL=HuggingFaceTB/SmolVLM2-2.2B-Instruct + +# Cache for local VLM (only used with --local-vision CLI flag) +HF_HOME=./cache + +# Hugging Face token for accessing datasets/models +HF_TOKEN= + +# Profile selection (ignored when LOCKED_PROFILE is set in config.py) +# REACHY_MINI_CUSTOM_PROFILE="example" + +# Skip loading .env if you prefer environment-only configuration. +# REACHY_MINI_SKIP_DOTENV=1 + +# Optional external profile/tool directories +# REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY=external_content/external_profiles +# REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY=external_content/external_tools + +# Optional: discover and auto-load all tools found in REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY, +# even if they are not listed in the selected profile's tools.txt. +# This is convenient for downloaded tools used with built-in/default profiles. +# AUTOLOAD_EXTERNAL_TOOLS=1 + +# Resend transactional email API for the send_email tool. +# Get a free API key at https://resend.com/api-keys (3000 emails/month free). +# Without RESEND_API_KEY, send_email writes to the in-memory outbox only +# (visible in the dashboard's Mailbox Out panel, but nothing actually leaves +# the robot). +# +# RESEND_FROM defaults to "onboarding@resend.dev" — Resend's sandbox sender +# that ONLY delivers to the email address registered on your Resend account. +# For production / arbitrary recipients, verify a domain at +# https://resend.com/domains and set RESEND_FROM to an address on that +# domain (e.g. "noreply@methdai.com"). +RESEND_API_KEY= +RESEND_FROM=onboarding@resend.dev + +# ---- Reception calendar (Google Calendar via iCal) ---- +# Set this to enable scheduled-visitor flow. The receptionist pulls today's +# appointments live from this URL (cached ~5 min). When unset, the bot +# serves walk-in visitors only — they say "I'm here to see X" and the bot +# routes via the employee directory (managed from the dashboard's +# Employees panel). There is no hardcoded demo schedule. +# +# To get a URL: in Google Calendar, create a calendar (e.g. "MethdAI +# Reception") -> Settings and sharing -> Integrate calendar -> +# "Public address in iCal format". Paste it below. +# +# Event title convention: " with " +# - "Rohan Verma with Mukul" +# - "Sara Khan with Priya — product demo follow-up" +# Host name matches the employee directory (employees.py); aliases work. +# An optional " — note" suffix after the host becomes the appointment note; +# alternatively put it in the event's DESCRIPTION field. +# RECEPTION_ICS_URL=https://calendar.google.com/calendar/ical/.../public/basic.ics + +# Timezone used to display iCal event times on the dashboard and to the LLM. +# Must be a valid IANA tz name. Defaults to Asia/Kolkata (pilot deployment +# is in India). Set this when the robot OS is in a different tz than your +# operators expect to see times in. Common values: +# Asia/Kolkata India (IST, UTC+5:30) +# America/New_York US East Coast +# Europe/London UK +# Asia/Tokyo Japan +# RECEPTION_TIMEZONE=Asia/Kolkata + +# ---- Privacy retention ---- +# Guest face crops in guests/*.png older than this many days are deleted +# at app startup. Set to 0 to disable (keep faces forever until FIFO +# capacity eviction kicks in at 100). +FACE_TTL_DAYS=30 +# Visit rows in visitor_log.db older than this many days are deleted at +# app startup. Set to 0 to disable (unbounded growth). +VISITOR_LOG_RETENTION_DAYS=90 + +# LBPH face-recognition strictness. LOWER = stricter (fewer false matches, +# more "I don't recognise you, please tell me your name" prompts). +# 50 - 75 recommended for production (default 75) +# 75 - 100 permissive (some lighting/angle variance OK) +# 100 - 110 old default — produces frequent false matches +# If returning guests stop being recognised, raise by 10 and re-test. +# If strangers get greeted as someone else, lower by 10. +FACE_LBPH_THRESHOLD=75 \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..e152ac7be09cd71383920c2cab6af5b2a79de9cf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,71 @@ +# Force LF line endings for all text files. Without this, Windows +# checkouts produce CRLF files that crash Python 3.12 when imported on +# Linux (observed: "from __future__ imports must occur at the +# beginning of the file" SyntaxError from background_tool_manager.py). +* text=auto eol=lf +*.py text eol=lf +*.txt text eol=lf +*.md text eol=lf +*.json text eol=lf +*.yaml text eol=lf +*.yml text eol=lf +*.toml text eol=lf +*.html text eol=lf +*.css text eol=lf +*.js text eol=lf +*.ics text eol=lf +*.sh text eol=lf +*.cfg text eol=lf +*.ini text eol=lf + +# Macro for all binary files that should use Git LFS. +[attr]lfs -text filter=lfs diff=lfs merge=lfs + +# Image +*.jpg lfs +*.jpeg lfs +*.png lfs +*.apng lfs +*.atsc lfs +*.gif lfs +*.bmp lfs +*.exr lfs +*.tga lfs +*.tiff lfs +*.tif lfs +*.iff lfs +*.pict lfs +*.dds lfs +*.xcf lfs +*.leo lfs +*.kra lfs +*.kpp lfs +*.clip lfs +*.webm lfs +*.webp lfs +*.svg lfs +*.svgz lfs +*.psd lfs +*.afphoto lfs +*.afdesign lfs +# Models +*.pth lfs +# Binaries +*.bin lfs +*.pkl lfs +*.pckl lfs +# 3D +*.ply lfs +*.vis lfs +*.db lfs +*.ply lfs +.git_disabled/lfs/objects/5a/63/5a63ac8802ff3542f01292c431c5278296880d74cd3580d219fcf4827bc235f9 filter=lfs diff=lfs merge=lfs -text +.git_disabled/lfs/objects/75/91/75914c3cb7af982e0b1c6369e25fc46d8c08a0ab5ad022240ae9c1a0d93967c3 filter=lfs diff=lfs merge=lfs -text +.git_disabled/lfs/objects/e9/7c/e97ca125a86bacdaa41c8dca88abd9ca746fd5c9391eda24249c012432b0219b filter=lfs diff=lfs merge=lfs -text +.git_disabled/objects/pack/pack-ba33ec9fbb4d88d9fd0f2be18721a74ddb3ca16f.pack filter=lfs diff=lfs merge=lfs -text +build/lib/reachy_mini_receptionist/images/reachymini_avatar.png filter=lfs diff=lfs merge=lfs -text +build/lib/reachy_mini_receptionist/images/user_avatar.png filter=lfs diff=lfs merge=lfs -text +docs/assets/reachy_mini_dance.gif filter=lfs diff=lfs merge=lfs -text +screenshot.png filter=lfs diff=lfs merge=lfs -text +src/reachy_mini_receptionist/images/reachymini_avatar.png filter=lfs diff=lfs merge=lfs -text +src/reachy_mini_receptionist/images/user_avatar.png filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a353d3db37e8a66a110cdcf544f60340da9164bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,78 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# Environment variables +.env + +# Build and distribution +build/ +dist/ +*.egg-info/ +.eggs/ + +# Testing +.pytest_cache/ +.coverage +.hypothesis/ +htmlcov/ +coverage.xml +*.cover + +# Linting and formatting +.ruff_cache/ +.mypy_cache/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Editor / IDE local settings (user-specific configuration) +.claude/ +.cursor/ +.vscode/ +.idea/ + +# Security +*.key +*.pem +*.crt +*.csr + +# Temporary files +tmp/ +*.log +cache/ + +# macOS +.DS_Store + +# Linux +*~ +.directory +.Trash-* +.nfs* + +# User-created personalities (managed by UI) +src/reachy_mini_receptionist/profiles/user_personalities/ + +# Runtime data (recreated on first run) +*.db +*.db-wal +*.db-shm +src/reachy_mini_receptionist/guests/*.png +.env.save +src/reachy_mini_receptionist.egg-info/ + +# Old git backup directories +.git_disabled/ diff --git a/.hfignore b/.hfignore new file mode 100644 index 0000000000000000000000000000000000000000..059dfe7245c834829c513553fce04f3280123a85 --- /dev/null +++ b/.hfignore @@ -0,0 +1,23 @@ +# Local environment and secrets +.env +src/reachy_mini_receptionist/.env + +# Python caches and build outputs +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +build/ +dist/ +*.egg-info/ + +# Local VCS artifacts that should never ship in published bundles +.git/ +.git_disabled/ + +# Local virtual environments +.venv/ +venv/ +ENV/ +env/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ecea948b51fc6c5e10a81c459b5f31bfd41c0ed --- /dev/null +++ b/README.md @@ -0,0 +1,152 @@ +--- +title: MethdAI Receptionist +emoji: 🤖 +colorFrom: indigo +colorTo: purple +sdk: gradio +app_file: src/reachy_mini_receptionist/main.py +pinned: false +license: apache-2.0 +short_description: Voice-driven receptionist for Reachy Mini — face recognition, real email, live ops dashboard. +--- + +# MethdAI Receptionist + +AI receptionist for the **Reachy Mini** robot. A visitor walks up, the bot greets them, asks who they're here to see, looks the host up in the directory, and emails the host that their guest has arrived. Returning visitors are recognized by face on their next visit. + +Built by **[MethdAI](https://methdai.com)**. + +--- + +## What it does + +- **Recognizes faces** — YuNet detection + LBPH recognition. New visitors get registered after they confirm their name. +- **Talks naturally** — Google Gemini Live (default) or OpenAI Realtime, voice-to-voice over the robot's mic/speaker. +- **Reads the calendar** — pulls today's schedule from a Google Calendar iCal feed. +- **Emails the host** — sends real notifications via Resend. +- **Logs every visit** — SQLite, exportable as CSV. +- **Configured from a browser** — no `.env` editing, no SSH for routine config. + +--- + +## Hardware required + +A real **[Reachy Mini](https://www.pollen-robotics.com/reachy-mini/)** robot from Pollen Robotics. The code expects the robot's camera, microphone, and speaker — there's no cloud-only mode. + +--- + +## Quick start + +### Option 1 — Install via the Reachy Mini Control app (recommended) + +1. Open Reachy Mini Control on your computer +2. **Install from Hugging Face** → search `methdai/reachy_mini_receptionist` → Install +3. Toggle the app **On** +4. Open the dashboard at `http://.local:7860/dashboard` +5. Follow the welcome banner — it tells you exactly which API keys to add and where to find them + +### Option 2 — Manual install (for development) + +```bash +# SSH into the robot +git clone git@github.com:methdai/reachy_mini_receptionist.git +cd reachy_mini_receptionist + +# Install the package in editable mode +/venvs/apps_venv/bin/pip install -e . + +# Make sure the Reachy Mini daemon is running +sudo systemctl status reachy-mini-daemon + +# Start the app +/venvs/apps_venv/bin/python -m reachy_mini_receptionist.main +``` + +Then open `http://localhost:7860/dashboard` in any browser on the same network. + +--- + +## Configuration + +**Everything is editable from the dashboard's Settings panel.** No SSH, no `.env` edits. + +The welcome banner on first launch tells you which keys are missing. Click each banner item → it scrolls you to the right field. + +### Required for full functionality + +| Setting | Why | Where to get it | +|---|---|---| +| `GEMINI_API_KEY` | Voice (default backend) | [aistudio.google.com/app/apikey](https://aistudio.google.com/app/apikey) — free tier works | +| `RESEND_API_KEY` | Send emails to hosts | [resend.com](https://resend.com) — free 3000 emails/month | +| `RECEPTION_ICS_URL` | Read today's appointments | Google Calendar → Settings → **Secret address in iCal format** | + +### Optional + +| Setting | Purpose | +|---|---| +| `VOICE_BACKEND` | Switch between `gemini` (default) and `openai` | +| `GEMINI_LIVE_VOICE` | Pick a voice — Puck, Charon, Kore, Aoede, etc. | +| `GEMINI_LIVE_MODEL` | Override the default Gemini Live model | +| `OPENAI_API_KEY` | Required only if `VOICE_BACKEND=openai` | +| `RESEND_FROM` | Sender address — defaults to Resend's sandbox sender (delivers only to your Resend account); set to `reception@yourdomain.com` after verifying your domain at Resend | +| `FACE_TTL_DAYS` | How long a registered face is remembered (default 90 days) | +| `VISITOR_LOG_RETENTION_DAYS` | How long visit records are kept (default 365 days) | + +--- + +## How it works + +**Vision** — `face_recognition_worker.py` runs YuNet on every frame, detects faces, and feeds crops to LBPH for recognition. + +**State machine** — `session_manager.py` tracks the current visitor (`idle → visitor_detected → recognized / asking_name → appointment_matched → notified`). `conversation_controller.py` decides what state to move to based on face events and tool results. + +**Voice + tools** — Either `gemini_live.py` or `openai_realtime.py` (chosen by `VOICE_BACKEND`) handles bi-directional audio with the LLM. Tools available to the LLM: +- `get_today_calendar` — fetch today's appointments +- `register_guest` — save a new visitor's face under a name (requires confirmation) +- `lookup_employee` — find a host in the directory +- `send_email` — notify the host + +**Persistence** — three SQLite databases (WAL mode): +- `employees.db` — the directory you edit in the dashboard +- `visitor_log.db` — every completed visit, exportable as CSV +- `guests/` directory — saved face crops, one PNG per visitor + +--- + +## Dashboard + +Open `http://:7860/dashboard` while the app is running. + +**Live view** — visit count, last visitor, live camera feed, face recognition status. +**Active Session** — what state the current visitor is in, what the bot heard, what cue it sent to the LLM. +**Today** — appointments from the calendar, known guests, recent outgoing emails. +**History** — full visitor log with CSV export and per-row delete. +**Employees** — add / edit / delete people the bot can notify. +**Settings** — every environment variable in one place. Toggle dark/light theme from the header. + +--- + +## Development + +```bash +# Install dev dependencies +/venvs/apps_venv/bin/pip install -e ".[dev]" + +# Run tests +pytest + +# Type-check +mypy src/ + +# Format / lint +ruff check src/ +ruff format src/ +``` + +The codebase is single-process Python 3.10+. Audio streams over WebRTC via `fastrtc`. The dashboard is a single static HTML file in `src/reachy_mini_receptionist/static/`. + +--- + +## License + +Apache-2.0 diff --git a/deploy/install_systemd.sh b/deploy/install_systemd.sh new file mode 100644 index 0000000000000000000000000000000000000000..08f7a84de95950cd5c60e14c643ef87daed5a968 --- /dev/null +++ b/deploy/install_systemd.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# +# Install (or upgrade) the MethdAI Receptionist systemd unit on the robot. +# Run on the robot itself (e.g. via ssh pollen@reachy-mini.local). +# +# Idempotent: re-running picks up edits to reachy-receptionist.service and +# restarts the live service. +# +# After install, useful commands: +# systemctl status reachy-receptionist +# journalctl -u reachy-receptionist -f +# sudo systemctl restart reachy-receptionist +# sudo systemctl disable --now reachy-receptionist +# +set -euo pipefail + +UNIT_NAME="reachy-receptionist.service" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SRC="$HERE/$UNIT_NAME" +DST="/etc/systemd/system/$UNIT_NAME" + +if [[ ! -f "$SRC" ]]; then + echo "✗ Unit file not found at $SRC" >&2 + exit 1 +fi + +# Re-elevate via `sudo bash ` so we don't depend on the script +# having the executable bit set (git-from-Windows often drops it, and we +# want `bash deploy/install_systemd.sh` to Just Work). +if [[ "$(id -u)" -ne 0 ]]; then + echo "↻ Re-running with sudo..." + exec sudo --preserve-env=HOME bash "$HERE/$(basename "${BASH_SOURCE[0]}")" "$@" +fi + +# Sanity check the runtime paths referenced by the unit file. Catching this +# now beats debugging a cryptic "ExecStart failed" later. +PYTHON_BIN="/venvs/apps_venv/bin/python" +PROJECT_DIR="/home/pollen/reachy_mini_receptionist" +if [[ ! -x "$PYTHON_BIN" ]]; then + echo "✗ Python interpreter not found at $PYTHON_BIN" >&2 + echo " Fix: install / locate the apps_venv before re-running." >&2 + exit 2 +fi +if [[ ! -d "$PROJECT_DIR" ]]; then + echo "✗ Project not found at $PROJECT_DIR" >&2 + exit 3 +fi + +echo "→ Installing $UNIT_NAME → $DST" +cp "$SRC" "$DST" +chmod 644 "$DST" + +echo "→ Reloading systemd" +systemctl daemon-reload + +echo "→ Enabling on boot" +systemctl enable "$UNIT_NAME" + +echo "→ Restarting service" +systemctl restart "$UNIT_NAME" + +# Brief wait so the status snapshot below shows a meaningful state +sleep 2 + +echo +echo "✓ Installed. Current status:" +echo "─────────────────────────────────────────" +systemctl --no-pager --lines=10 status "$UNIT_NAME" || true +echo "─────────────────────────────────────────" +echo +echo "Follow logs: journalctl -u $UNIT_NAME -f" +echo "Manual restart: sudo systemctl restart $UNIT_NAME" +echo "Stop: sudo systemctl stop $UNIT_NAME" +echo "Disable: sudo systemctl disable --now $UNIT_NAME" diff --git a/deploy/reachy-receptionist.service b/deploy/reachy-receptionist.service new file mode 100644 index 0000000000000000000000000000000000000000..f8e5023fd9056806926cca8a332b0749778ccc45 --- /dev/null +++ b/deploy/reachy-receptionist.service @@ -0,0 +1,39 @@ +[Unit] +Description=MethdAI Receptionist (Reachy Mini) +Documentation=https://github.com/mukul-chauhan-methdai/reachy_mini_receptionist +After=network-online.target +Wants=network-online.target +# If the receptionist can't come up after 10 attempts in 10 minutes, stop +# retrying so we don't hammer the OpenAI / Resend / camera APIs on a +# permanent failure. +StartLimitBurst=10 +StartLimitIntervalSec=600 + +[Service] +Type=simple +User=pollen +Group=pollen +WorkingDirectory=/home/pollen/reachy_mini_receptionist + +# Load .env into the service environment. The "-" prefix makes the file +# optional — missing .env logs a warning but doesn't block start. +EnvironmentFile=-/home/pollen/reachy_mini_receptionist/.env + +# Best-effort: wake the reachy_mini daemon before launching the app. +# Retries every 3s for up to 30s. We always exit 0 so a failed wake +# doesn't kill the unit before ExecStart; the app itself will crash and +# systemd will restart it if the daemon is genuinely down. +ExecStartPre=/bin/bash -c 'for i in $(seq 1 10); do curl -fsS -X POST "http://localhost:8000/api/daemon/start?wake_up=true" >/dev/null && exit 0; sleep 3; done; exit 0' + +ExecStart=/venvs/apps_venv/bin/python -m reachy_mini_receptionist.main + +Restart=on-failure +RestartSec=5 + +# Capture stdout/stderr in the journal — view with: +# journalctl -u reachy-receptionist -f +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/docs/assets/conversation_app_arch.svg b/docs/assets/conversation_app_arch.svg new file mode 100644 index 0000000000000000000000000000000000000000..c0a51cf50856b4979491d107902f0b37fb6d2f8c --- /dev/null +++ b/docs/assets/conversation_app_arch.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0013aac9cbe5f78a2aed3ed4de5fab5c3afe36ff72950ac97e64bd5db462e3b9 +size 123931 diff --git a/docs/assets/reachy_mini_dance.gif b/docs/assets/reachy_mini_dance.gif new file mode 100644 index 0000000000000000000000000000000000000000..d801b57f4de4bcba30589b44522f99975ba7bdca --- /dev/null +++ b/docs/assets/reachy_mini_dance.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75914c3cb7af982e0b1c6369e25fc46d8c08a0ab5ad022240ae9c1a0d93967c3 +size 3930400 diff --git a/docs/scheme.mmd b/docs/scheme.mmd new file mode 100644 index 0000000000000000000000000000000000000000..c277c7f7a0965222df0669582f4efc5bb67746f8 --- /dev/null +++ b/docs/scheme.mmd @@ -0,0 +1,63 @@ +--- +config: + layout: dagre + flowchart: + htmlLabels: true +--- +flowchart TB + User(["User
Person interacting with system"]) + -- audio stream --> + UI@{ label: "UI Layer
Gradio/Console" } + + UI -- audio stream --> + OpenAI@{ label: "gpt-realtime API
Audio+Tool Calls+Vision" } + + OpenAI -- audio stream --> + Motion@{ label: "Motion Control
Audio Sync + Tracking" } + + OpenAI -- tool calls --> + Handlers@{ label: "Tool Layer
Built-in tools + profile-local tools
+ external tools (optional)" } + + Profiles@{ label: "Selected Profile
built-in or external
instructions.txt + tools.txt" } + + Profiles -- defines enabled tools --> Handlers + + Handlers -- movement + requests --> Motion + + Handlers -- camera frames, head tracking --> + Camera@{ label: "Camera Worker
Frame Buffer + Head Tracking" } + + Handlers -. image for + analysis .-> OpenAI + + Camera -- head tracking --> Motion + + Camera -. frames .-> + Vision@{ label: "Vision Processor
Local VLM (optional)" } + + Vision -. description .-> Handlers + + Robot@{ label: "reachy_mini
Robot Control Library" } + -- camera + frames --> Camera + + Motion -- commands --> Robot + + Handlers -- results --> OpenAI + + User:::userStyle + UI:::uiStyle + OpenAI:::aiStyle + Motion:::coreStyle + Profiles:::toolStyle + Handlers:::toolStyle + Camera:::coreStyle + Vision:::aiStyle + Robot:::hardwareStyle + classDef userStyle fill:#e1f5fe,stroke:#01579b,stroke-width:3px + classDef uiStyle fill:#b3e5fc,stroke:#0277bd,stroke-width:2px + classDef aiStyle fill:#e1bee7,stroke:#7b1fa2,stroke-width:3px + classDef coreStyle fill:#fff9c4,stroke:#f57f17,stroke-width:2px + classDef hardwareStyle fill:#ef9a9a,stroke:#c62828,stroke-width:3px + classDef toolStyle fill:#fffde7,stroke:#f9a825,stroke-width:1px diff --git a/external_content/external_profiles/starter_profile/instructions.txt b/external_content/external_profiles/starter_profile/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d2a1c2c24fd7c37aa3a573a0ece54b424f7511c --- /dev/null +++ b/external_content/external_profiles/starter_profile/instructions.txt @@ -0,0 +1,6 @@ +You are a helpful Reachy Mini assistant running from an external profile. + +When asked to demonstrate your custom greeting, use the `starter_custom_tool` tool. +You can also dance and show emotions like the built-in profiles. + +Be friendly and concise, and explain that you're using an external profile/tool setup when asked about yourself. diff --git a/external_content/external_profiles/starter_profile/tools.txt b/external_content/external_profiles/starter_profile/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..254a42f6135e5157662610f3a8b7ec24c330812c --- /dev/null +++ b/external_content/external_profiles/starter_profile/tools.txt @@ -0,0 +1,9 @@ +# This file is an explicit allow-list. +# Every tool name listed below must be either: +# - a built-in tool from src/reachy_mini_receptionist/tools/ +# - or an external tool file in TOOLS_DIRECTORY (e.g. external_tools/starter_custom_tool.py) + +get_today_calendar +register_guest +send_email +starter_custom_tool diff --git a/external_content/external_tools/starter_custom_tool.py b/external_content/external_tools/starter_custom_tool.py new file mode 100644 index 0000000000000000000000000000000000000000..3d68e053ac8e5c6c38b87c577484951641f0d7a4 --- /dev/null +++ b/external_content/external_tools/starter_custom_tool.py @@ -0,0 +1,33 @@ +"""Example external tool implementation.""" + +import logging +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + + +class StarterCustomTool(Tool): + """Placeholder custom tool - demonstrates external tool loading.""" + + name = "starter_custom_tool" + description = "A placeholder custom tool loaded from outside the library" + parameters_schema = { + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "Optional message to include in the response", + }, + }, + "required": [], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Execute the placeholder tool.""" + message = kwargs.get("message", "Hello from custom tool!") + logger.info(f"Tool call: starter_custom_tool message={message}") + + return {"status": "success", "message": message} diff --git a/index.html b/index.html new file mode 100644 index 0000000000000000000000000000000000000000..8d204d24091cf11d1907327733edb018d8f4d481 --- /dev/null +++ b/index.html @@ -0,0 +1,141 @@ + + + + + + + Reachy Mini AI Receptionist + + + + + + + +
+
+
+ + Reachy Mini +
+
Realtime voice · Vision aware · Expressive motion
+
+
+
+

AI Receptionist

+

+ Face-aware receptionist with tool-calling automation. + Live demo +

+

+ A camera-aware front-desk assistant for Reachy Mini. Greet visitors naturally, register guests, check appointments, and log handoff actions from a single dashboard. +

+

+ Built for reception workflows: realtime face detection to know when someone is present, face recognition to personalize interactions, and structured tool calls so the AI can register guests, query appointments, and trigger handoff actions reliably. +

+
+ Explore features + See how it feels +
+
+ Realtime face detection + Visitor face recognition + Model tool-calling workflows + Low-latency voice + dashboard ops +
+
+
+
+ Reachy Mini AI Receptionist screenshot +

Reachy Mini can greet, identify, and assist visitors with receptionist-specific context.

+
+ +
+

Watch on YouTube

+
+
+
+
+ +
+
+

What’s inside

+

All-in-one receptionist layer for your robot

+

+ The app blends realtime speech, vision, and workflow tools so Reachy Mini can run a front desk flow. +

+
+
+
+ 🎤 +

Natural voice chat

+

Talk freely and get fast, high-quality replies powered by realtime models.

+
+
+ 🎥 +

Face-aware onboarding

+

Recognize known visitors, register new guests, and keep the latest face context synced with the conversation.

+
+
+ 💃 +

Expressive motion

+

Use subtle head and antenna gestures during welcome and registration interactions.

+
+
+ 🧠 +

Calendar-aware assistance

+

Use appointment context to welcome guests on time and guide follow-up actions.

+
+
+ 🌐 +

Ready for your setup

+

Works in console mode or web mode with dashboard APIs for quick operator visibility.

+
+
+
+ +
+
+
+

How it feels

+

From greeting to handoff in seconds

+
    +
  • 👋 Greet visitors naturally with low-latency voice conversation.
    • +
  • 👀 Use camera context to identify known guests or register new ones.
    • +
  • 📅 Check appointment context and respond with relevant timing cues.
    • +
  • 📨 Log handoff actions in the outbox for host follow-up.
    • +
+
+
+

Where it shines

+

Great for offices, demos, and guided reception flows

+

+ Show how Reachy Mini can handle repeatable visitor workflows while staying expressive and conversational. It is ideal for front-desk demos, events, and product showcases. +

+
+ Guest recognition + Calendar check + Outbox logging + Dashboard APIs + Realtime conversation +
+
+
+
+ + + + + + diff --git a/plan.md b/plan.md new file mode 100644 index 0000000000000000000000000000000000000000..bffb7f20f3c1bf44109c6b96bef7969a6d880efb --- /dev/null +++ b/plan.md @@ -0,0 +1,89 @@ +# Reachy Mini AI Receptionist — Plan + +## What This App Does + +An interactive AI receptionist that: + +1. Talks to visitors via OpenAI Realtime API (low-latency speech in/out). +2. Monitors camera frames for face detection/recognition in a background worker. +3. Stores guest face crops as PNG files in a persistent `guests/` directory. +4. Uses a hardcoded POC calendar for appointment context. +5. Exposes a dashboard with live video, guests, calendar, outbox, and debug logs. + +Start command: + +```bash +python -m reachy_mini_receptionist.main +``` + +--- + +## Architecture + +The receptionist extends the conversation base stack and adds receptionist-specific modules: + +```text +src/reachy_mini_receptionist/ +├── face_db.py # File-based face store (PNG per guest in guests/) +├── face_recognition_worker.py # Background detection/recognition + event emission +├── calendar_data.py # Hardcoded appointment data +├── main.py # App entrypoint + dashboard API mounting +└── tools/ + ├── get_today_calendar.py + ├── register_guest.py + ├── send_email.py + └── check_current_face.py # Legacy compatibility path +``` + +--- + +## Key Design Decisions + +### 1) Face database is file-based + +- Guests are stored as grayscale PNG crops in `guests/`. +- `FaceDatabase` enforces capacity with FIFO-style eviction when full. +- No SQL database is required. + +### 2) Face recognition runs in a background worker + +- `FaceRecognitionWorker` runs independently from the realtime audio loop. +- Worker state is consumed by tools and dashboard endpoints. +- Stable face transitions emit context events to the model. + +### 3) Calendar is static for POC + +- `calendar_data.py` returns hardcoded appointments. +- Easy to swap later for Google/Microsoft calendar integrations. + +### 4) Dashboard API is mounted in-app + +- `GET /dashboard` serves the receptionist dashboard. +- `GET /video_feed` streams annotated MJPEG. +- `GET /api/guests`, `/api/calendar`, `/api/outbox`, `/api/face_status`, `/api/logs` expose app state. + +### 5) Profile is intentionally locked + +- `LOCKED_PROFILE` is set to `_reachy_mini_receptionist_locked_profile` in `config.py`. +- Current locked tool allow-list is: + - `get_today_calendar` + - `register_guest` + - `send_email` + +--- + +## Face Context Event Behavior + +The receptionist flow is push-based: + +- Stable face transitions are emitted by `FaceRecognitionWorker`. +- `OpenaiRealtimeHandler` injects these as context-only user items. +- No automatic `response.create` is triggered by these face context updates. + +--- + +## Known Notes + +- With `--no-camera`, recognition and registration tools cannot operate. +- Output language behavior is controlled by profile instructions. +- If profile/tool loading fails, the app can fall back to default model behavior; monitor startup logs. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..d052be0f489c512cec7df860184ff692549a6eb5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,81 @@ +[build-system] +requires = [ "setuptools",] +build-backend = "setuptools.build_meta" + +[project] +name = "reachy_mini_receptionist" +version = "0.3.0" +description = "Reachy Mini AI receptionist app with realtime voice, guest recognition, and dashboard tools." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ "aiortc>=1.13.0", "fastrtc>=0.0.34", "gradio==5.50.1.dev1", "huggingface-hub==1.3.0", "opencv-contrib-python>=4.8.0", "python-dotenv", "openai>=2.1", "google-genai>=1.40", "reachy_mini_dances_library", "reachy_mini_toolbox", "reachy-mini>=1.5.0", "eclipse-zenoh~=1.7.0", "gradio_client>=1.13.3", "numpy>=1.24", "httpx>=0.27", "icalendar>=5.0",] +license = "Apache-2.0" +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/pollen-robotics/reachy_mini" +Repository = "https://github.com/pollen-robotics/reachy_mini" +[[project.authors]] +name = "Pollen Robotics" +email = "contact@pollen-robotics.com" + +[dependency-groups] +dev = [ "pytest", "pytest-asyncio", "ruff==0.12.0", "mypy==1.18.2", "pre-commit", "types-requests", "python-semantic-release>=10.5.3",] + +[project.optional-dependencies] +local_vision = [ "torch>=2.1", "transformers==5.0.0rc2", "num2words",] +yolo_vision = [ "ultralytics", "supervision",] +mediapipe_vision = [ "mediapipe==0.10.14",] +all_vision = [ "torch>=2.1", "transformers==5.0.0rc2", "num2words", "ultralytics", "supervision", "mediapipe==0.10.14",] + +[project.scripts] +reachy-mini-receptionist = "reachy_mini_receptionist.main:main" + +[tool.setuptools] +include-package-data = true + +[tool.ruff] +line-length = 119 +exclude = [ ".venv", "dist", "build", "**/__pycache__", "*.egg-info", ".mypy_cache", ".pytest_cache",] + +[tool.mypy] +python_version = "3.12" +files = [ "src/",] +ignore_missing_imports = true +strict = true +show_error_codes = true +warn_unused_ignores = true + +[project.entry-points.reachy_mini_apps] +reachy_mini_receptionist = "reachy_mini_receptionist.main:ReachyMiniReceptionist" + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools.package-data] +reachy_mini_receptionist = [ "images/*", "static/*", ".env.example", "profiles/**/*.txt", "prompts/**/*.txt",] + +[tool.ruff.lint] +select = [ "E", "F", "W", "I", "C4", "D",] +ignore = [ "E501", "D100", "D203", "D213",] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" + +[tool.setuptools.packages.find] +where = [ "src",] + +[tool.ruff.lint.isort] +length-sort = true +lines-after-imports = 2 +no-lines-before = [ "standard-library", "local-folder",] +known-local-folder = [ "reachy_mini_receptionist",] +known-first-party = [ "reachy_mini", "reachy_mini_dances_library", "reachy_mini_toolbox",] +split-on-trailing-comma = true diff --git a/screenshot.png b/screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..97571d3aa0bd69154641ab88f309b56d9a819b43 --- /dev/null +++ b/screenshot.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:320d75c3ccc1f262d9bbfc98577d3883a0dde498860f69c2eee4c92810498f8c +size 1662120 diff --git a/scripts/gemini_live_smoke.py b/scripts/gemini_live_smoke.py new file mode 100644 index 0000000000000000000000000000000000000000..b22afa95898b5a9089923521dcd991bc959c92a6 --- /dev/null +++ b/scripts/gemini_live_smoke.py @@ -0,0 +1,96 @@ +"""Minimal Gemini Live smoke test — isolates SDK behavior from app. + +Reads GEMINI_API_KEY from environment, connects to the Live API with +the configured model, sends a single text turn, prints every event +received until session closes or 30s timeout. + +Use: + GEMINI_API_KEY=... /venvs/apps_venv/bin/python scripts/gemini_live_smoke.py + GEMINI_API_KEY=... GEMINI_LIVE_MODEL=gemini-2.0-flash-live-001 /venvs/apps_venv/bin/python scripts/gemini_live_smoke.py +""" +from __future__ import annotations + +import asyncio +import os +import sys + + +async def main() -> int: + key = os.environ.get("GEMINI_API_KEY", "").strip() + if not key: + print("ERROR: GEMINI_API_KEY not set", file=sys.stderr) + return 1 + + model = os.environ.get("GEMINI_LIVE_MODEL", "gemini-2.5-flash-native-audio-latest") + print(f"[smoke] model={model}") + + try: + from google import genai + except ImportError as e: + print(f"ERROR: google-genai not installed: {e}", file=sys.stderr) + return 1 + + print(f"[smoke] google-genai version={getattr(genai, '__version__', '?')}") + + client = genai.Client(api_key=key, http_options={"api_version": "v1beta"}) + + # Native-audio models REQUIRE AUDIO modality. The 1007 error + # "Cannot extract voices from a non-audio request" confirms this. + config = { + "response_modalities": ["AUDIO"], + } + try: + async with client.aio.live.connect(model=model, config=config) as session: + print("[smoke] connected; sending one text turn (turn_complete=True)...") + await session.send_client_content( + turns=[{"role": "user", "parts": [{"text": "Say hello in one short friendly sentence."}]}], + turn_complete=True, + ) + + event_count = 0 + audio_bytes_total = 0 + try: + async with asyncio.timeout(30): + async for resp in session.receive(): + event_count += 1 + text = getattr(resp, "text", None) + data = getattr(resp, "data", None) + if data: + audio_bytes_total += len(data) + sc = getattr(resp, "server_content", None) + tc = getattr(sc, "turn_complete", None) if sc else None + model_turn = getattr(sc, "model_turn", None) if sc else None + mt_parts_summary = "" + if model_turn is not None: + parts = getattr(model_turn, "parts", None) or [] + mt_parts_summary = f" model_turn.parts={len(parts)}" + for i, p in enumerate(parts[:3]): + ip = getattr(p, "inline_data", None) + tp = getattr(p, "text", None) + th = getattr(p, "thought", None) + print( + f"[smoke] part {i}: text={tp!r}, " + f"inline_data={'<%d bytes>' % len(getattr(ip, 'data', b'')) if ip else None}, " + f"thought={th}" + ) + print( + f"[smoke] event #{event_count}: text={text!r}, " + f"data={'<%d bytes>' % len(data) if data else None}, " + f"turn_complete={tc}{mt_parts_summary}" + ) + if tc: + print("[smoke] turn_complete=True — exiting receive() loop") + break + except asyncio.TimeoutError: + print(f"[smoke] timed out after 30s, events={event_count}, audio_total={audio_bytes_total} bytes") + print(f"[smoke] done. total events={event_count}, total_audio_bytes={audio_bytes_total}") + except Exception as e: + import traceback + print(f"[smoke] CONNECTION ERROR: {e}") + traceback.print_exc() + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/list_gemini_live_models.py b/scripts/list_gemini_live_models.py new file mode 100644 index 0000000000000000000000000000000000000000..f10f55f5bc933469a96dcea89c5d595d53c9105d --- /dev/null +++ b/scripts/list_gemini_live_models.py @@ -0,0 +1,50 @@ +"""List all Gemini models on this API key that support bidiGenerateContent. + +These are the models you can put in GEMINI_LIVE_MODEL. Anything not listed +here will 1008 at connect time. + +Use: + GEMINI_API_KEY=... /venvs/apps_venv/bin/python scripts/list_gemini_live_models.py +""" +from __future__ import annotations + +import os +import sys + +import httpx + + +def main() -> int: + key = os.environ.get("GEMINI_API_KEY", "").strip() + if not key: + print("ERROR: GEMINI_API_KEY not set", file=sys.stderr) + return 1 + + url = f"https://generativelanguage.googleapis.com/v1beta/models?key={key}&pageSize=200" + try: + resp = httpx.get(url, timeout=15.0) + resp.raise_for_status() + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + return 1 + + data = resp.json() + models = data.get("models", []) + live_models = [] + for m in models: + methods = m.get("supportedGenerationMethods") or [] + if "bidiGenerateContent" in methods: + live_models.append(m.get("name", "?").replace("models/", "")) + + if not live_models: + print("(no Live-capable models on this key)") + return 0 + + print(f"Live-capable models on this key ({len(live_models)}):\n") + for name in sorted(live_models): + print(f" {name}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/reachy_mini_receptionist/__init__.py b/src/reachy_mini_receptionist/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..60dc15c497e49cbeb8687afb06a62339c297e2b5 --- /dev/null +++ b/src/reachy_mini_receptionist/__init__.py @@ -0,0 +1 @@ +"""Nothing (for ruff).""" diff --git a/src/reachy_mini_receptionist/audio/__init__.py b/src/reachy_mini_receptionist/audio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..60dc15c497e49cbeb8687afb06a62339c297e2b5 --- /dev/null +++ b/src/reachy_mini_receptionist/audio/__init__.py @@ -0,0 +1 @@ +"""Nothing (for ruff).""" diff --git a/src/reachy_mini_receptionist/audio/head_wobbler.py b/src/reachy_mini_receptionist/audio/head_wobbler.py new file mode 100644 index 0000000000000000000000000000000000000000..4adcb4c638abea0594186fe37b94f492d48ff31b --- /dev/null +++ b/src/reachy_mini_receptionist/audio/head_wobbler.py @@ -0,0 +1,181 @@ +"""Moves head given audio samples.""" + +import time +import queue +import base64 +import logging +import threading +from typing import Tuple +from collections.abc import Callable + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini_receptionist.audio.speech_tapper import HOP_MS, SwayRollRT + + +SAMPLE_RATE = 24000 +MOVEMENT_LATENCY_S = 0.2 # seconds between audio and robot movement +logger = logging.getLogger(__name__) + + +class HeadWobbler: + """Converts audio deltas (base64) into head movement offsets.""" + + def __init__(self, set_speech_offsets: Callable[[Tuple[float, float, float, float, float, float]], None]) -> None: + """Initialize the head wobbler.""" + self._apply_offsets = set_speech_offsets + self._base_ts: float | None = None + self._hops_done: int = 0 + + self.audio_queue: "queue.Queue[Tuple[int, int, NDArray[np.int16]]]" = queue.Queue() + self.sway = SwayRollRT() + + # Synchronization primitives + self._state_lock = threading.Lock() + self._sway_lock = threading.Lock() + self._generation = 0 + + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + + def feed(self, delta_b64: str) -> None: + """Thread-safe: push audio into the consumer queue.""" + buf = np.frombuffer(base64.b64decode(delta_b64), dtype=np.int16).reshape(1, -1) + with self._state_lock: + generation = self._generation + self.audio_queue.put((generation, SAMPLE_RATE, buf)) + + def start(self) -> None: + """Start the head wobbler loop in a thread.""" + self._stop_event.clear() + self._thread = threading.Thread(target=self.working_loop, daemon=True) + self._thread.start() + logger.debug("Head wobbler started") + + def stop(self) -> None: + """Stop the head wobbler loop.""" + self._stop_event.set() + if self._thread is not None: + self._thread.join() + logger.debug("Head wobbler stopped") + + def working_loop(self) -> None: + """Convert audio deltas into head movement offsets.""" + hop_dt = HOP_MS / 1000.0 + + logger.debug("Head wobbler thread started") + while not self._stop_event.is_set(): + queue_ref = self.audio_queue + try: + chunk_generation, sr, chunk = queue_ref.get_nowait() # (gen, sr, data) + except queue.Empty: + # avoid while to never exit + time.sleep(MOVEMENT_LATENCY_S) + continue + + try: + with self._state_lock: + current_generation = self._generation + if chunk_generation != current_generation: + continue + + if self._base_ts is None: + with self._state_lock: + if self._base_ts is None: + self._base_ts = time.monotonic() + + pcm = np.asarray(chunk).squeeze(0) + with self._sway_lock: + results = self.sway.feed(pcm, sr) + + i = 0 + while i < len(results): + with self._state_lock: + if self._generation != current_generation: + break + base_ts = self._base_ts + hops_done = self._hops_done + + if base_ts is None: + base_ts = time.monotonic() + with self._state_lock: + if self._base_ts is None: + self._base_ts = base_ts + hops_done = self._hops_done + + target = base_ts + MOVEMENT_LATENCY_S + hops_done * hop_dt + now = time.monotonic() + + if now - target >= hop_dt: + lag_hops = int((now - target) / hop_dt) + drop = min(lag_hops, len(results) - i - 1) + if drop > 0: + with self._state_lock: + self._hops_done += drop + hops_done = self._hops_done + i += drop + continue + + if target > now: + time.sleep(target - now) + with self._state_lock: + if self._generation != current_generation: + break + + r = results[i] + offsets = ( + r["x_mm"] / 1000.0, + r["y_mm"] / 1000.0, + r["z_mm"] / 1000.0, + r["roll_rad"], + r["pitch_rad"], + r["yaw_rad"], + ) + + with self._state_lock: + if self._generation != current_generation: + break + + self._apply_offsets(offsets) + + with self._state_lock: + self._hops_done += 1 + i += 1 + finally: + queue_ref.task_done() + logger.debug("Head wobbler thread exited") + + ''' + def drain_audio_queue(self) -> None: + """Empty the audio queue.""" + try: + while True: + self.audio_queue.get_nowait() + except QueueEmpty: + pass + ''' + + def reset(self) -> None: + """Reset the internal state.""" + with self._state_lock: + self._generation += 1 + self._base_ts = None + self._hops_done = 0 + + # Drain any queued audio chunks from previous generations + drained_any = False + while True: + try: + _, _, _ = self.audio_queue.get_nowait() + except queue.Empty: + break + else: + drained_any = True + self.audio_queue.task_done() + + with self._sway_lock: + self.sway.reset() + + if drained_any: + logger.debug("Head wobbler queue drained during reset") diff --git a/src/reachy_mini_receptionist/audio/speech_tapper.py b/src/reachy_mini_receptionist/audio/speech_tapper.py new file mode 100644 index 0000000000000000000000000000000000000000..3f4a2f16f9c9dae14946f6e0b50db8d2bb0b08dd --- /dev/null +++ b/src/reachy_mini_receptionist/audio/speech_tapper.py @@ -0,0 +1,268 @@ +from __future__ import annotations +import math +from typing import Any, Dict, List +from itertools import islice +from collections import deque + +import numpy as np +from numpy.typing import NDArray + + +# Tunables +SR = 16_000 +FRAME_MS = 20 +HOP_MS = 50 + +SWAY_MASTER = 1.5 +SENS_DB_OFFSET = +4.0 +VAD_DB_ON = -35.0 +VAD_DB_OFF = -45.0 +VAD_ATTACK_MS = 40 +VAD_RELEASE_MS = 250 +ENV_FOLLOW_GAIN = 0.65 + +SWAY_F_PITCH = 2.2 +SWAY_A_PITCH_DEG = 4.5 +SWAY_F_YAW = 0.6 +SWAY_A_YAW_DEG = 7.5 +SWAY_F_ROLL = 1.3 +SWAY_A_ROLL_DEG = 2.25 +SWAY_F_X = 0.35 +SWAY_A_X_MM = 4.5 +SWAY_F_Y = 0.45 +SWAY_A_Y_MM = 3.75 +SWAY_F_Z = 0.25 +SWAY_A_Z_MM = 2.25 + +SWAY_DB_LOW = -46.0 +SWAY_DB_HIGH = -18.0 +LOUDNESS_GAMMA = 0.9 +SWAY_ATTACK_MS = 50 +SWAY_RELEASE_MS = 250 + +# Derived +FRAME = int(SR * FRAME_MS / 1000) +HOP = int(SR * HOP_MS / 1000) +ATTACK_FR = max(1, int(VAD_ATTACK_MS / HOP_MS)) +RELEASE_FR = max(1, int(VAD_RELEASE_MS / HOP_MS)) +SWAY_ATTACK_FR = max(1, int(SWAY_ATTACK_MS / HOP_MS)) +SWAY_RELEASE_FR = max(1, int(SWAY_RELEASE_MS / HOP_MS)) + + +def _rms_dbfs(x: NDArray[np.float32]) -> float: + """Root-mean-square in dBFS for float32 mono array in [-1,1].""" + # numerically stable rms (avoid overflow) + x = x.astype(np.float32, copy=False) + rms = np.sqrt(np.mean(x * x, dtype=np.float32) + 1e-12, dtype=np.float32) + return float(20.0 * math.log10(float(rms) + 1e-12)) + + +def _loudness_gain(db: float, offset: float = SENS_DB_OFFSET) -> float: + """Normalize dB into [0,1] with gamma; clipped to [0,1].""" + t = (db + offset - SWAY_DB_LOW) / (SWAY_DB_HIGH - SWAY_DB_LOW) + if t < 0.0: + t = 0.0 + elif t > 1.0: + t = 1.0 + return t**LOUDNESS_GAMMA if LOUDNESS_GAMMA != 1.0 else t + + +def _to_float32_mono(x: NDArray[Any]) -> NDArray[np.float32]: + """Convert arbitrary PCM array to float32 mono in [-1,1]. + + Accepts shapes: (N,), (1,N), (N,1), (C,N), (N,C). + """ + a = np.asarray(x) + if a.ndim == 0: + return np.zeros(0, dtype=np.float32) + + # If 2D, decide which axis is channels (prefer small first dim) + if a.ndim == 2: + # e.g., (channels, samples) if channels is small (<=8) + if a.shape[0] <= 8 and a.shape[0] <= a.shape[1]: + a = np.mean(a, axis=0) + else: + a = np.mean(a, axis=1) + elif a.ndim > 2: + a = np.mean(a.reshape(a.shape[0], -1), axis=0) + + # Now 1D, cast/scale + if np.issubdtype(a.dtype, np.floating): + return a.astype(np.float32, copy=False) + # integer PCM + info = np.iinfo(a.dtype) + scale = float(max(-info.min, info.max)) + return a.astype(np.float32) / (scale if scale != 0.0 else 1.0) + + +def _resample_linear(x: NDArray[np.float32], sr_in: int, sr_out: int) -> NDArray[np.float32]: + """Lightweight linear resampler for short buffers.""" + if sr_in == sr_out or x.size == 0: + return x + # guard tiny sizes + n_out = int(round(x.size * sr_out / sr_in)) + if n_out <= 1: + return np.zeros(0, dtype=np.float32) + t_in = np.linspace(0.0, 1.0, num=x.size, dtype=np.float32, endpoint=True) + t_out = np.linspace(0.0, 1.0, num=n_out, dtype=np.float32, endpoint=True) + return np.interp(t_out, t_in, x).astype(np.float32, copy=False) + + +class SwayRollRT: + """Feed audio chunks → per-hop sway outputs. + + Usage: + rt = SwayRollRT() + rt.feed(pcm_int16_or_float, sr) -> List[dict] + """ + + def __init__(self, rng_seed: int = 7): + """Initialize state.""" + self._seed = int(rng_seed) + self.samples: deque[float] = deque(maxlen=10 * SR) # sliding window for VAD/env + self.carry: NDArray[np.float32] = np.zeros(0, dtype=np.float32) + + self.vad_on = False + self.vad_above = 0 + self.vad_below = 0 + + self.sway_env = 0.0 + self.sway_up = 0 + self.sway_down = 0 + + rng = np.random.default_rng(self._seed) + self.phase_pitch = float(rng.random() * 2 * math.pi) + self.phase_yaw = float(rng.random() * 2 * math.pi) + self.phase_roll = float(rng.random() * 2 * math.pi) + self.phase_x = float(rng.random() * 2 * math.pi) + self.phase_y = float(rng.random() * 2 * math.pi) + self.phase_z = float(rng.random() * 2 * math.pi) + self.t = 0.0 + + def reset(self) -> None: + """Reset state (VAD/env/buffers/time) but keep initial phases/seed.""" + self.samples.clear() + self.carry = np.zeros(0, dtype=np.float32) + self.vad_on = False + self.vad_above = 0 + self.vad_below = 0 + self.sway_env = 0.0 + self.sway_up = 0 + self.sway_down = 0 + self.t = 0.0 + + def feed(self, pcm: NDArray[Any], sr: int | None) -> List[Dict[str, float]]: + """Stream in PCM chunk. Returns a list of sway dicts, one per hop (HOP_MS). + + Args: + pcm: np.ndarray, shape (N,) or (C,N)/(N,C); int or float. + sr: sample rate of `pcm` (None -> assume SR). + + """ + sr_in = SR if sr is None else int(sr) + x = _to_float32_mono(pcm) + if x.size == 0: + return [] + if sr_in != SR: + x = _resample_linear(x, sr_in, SR) + if x.size == 0: + return [] + + # append to carry and consume fixed HOP chunks + if self.carry.size: + self.carry = np.concatenate([self.carry, x]) + else: + self.carry = x + + out: List[Dict[str, float]] = [] + + while self.carry.size >= HOP: + hop = self.carry[:HOP] + remaining: NDArray[np.float32] = self.carry[HOP:] + self.carry = remaining + + # keep sliding window for VAD/env computation + # (deque accepts any iterable; list() for small HOP is fine) + self.samples.extend(hop.tolist()) + if len(self.samples) < FRAME: + self.t += HOP_MS / 1000.0 + continue + + frame = np.fromiter( + islice(self.samples, len(self.samples) - FRAME, len(self.samples)), + dtype=np.float32, + count=FRAME, + ) + db = _rms_dbfs(frame) + + # VAD with hysteresis + attack/release + if db >= VAD_DB_ON: + self.vad_above += 1 + self.vad_below = 0 + if not self.vad_on and self.vad_above >= ATTACK_FR: + self.vad_on = True + elif db <= VAD_DB_OFF: + self.vad_below += 1 + self.vad_above = 0 + if self.vad_on and self.vad_below >= RELEASE_FR: + self.vad_on = False + + if self.vad_on: + self.sway_up = min(SWAY_ATTACK_FR, self.sway_up + 1) + self.sway_down = 0 + else: + self.sway_down = min(SWAY_RELEASE_FR, self.sway_down + 1) + self.sway_up = 0 + + up = self.sway_up / SWAY_ATTACK_FR + down = 1.0 - (self.sway_down / SWAY_RELEASE_FR) + target = up if self.vad_on else down + self.sway_env += ENV_FOLLOW_GAIN * (target - self.sway_env) + # clamp + if self.sway_env < 0.0: + self.sway_env = 0.0 + elif self.sway_env > 1.0: + self.sway_env = 1.0 + + loud = _loudness_gain(db) * SWAY_MASTER + env = self.sway_env + self.t += HOP_MS / 1000.0 + + # oscillators + pitch = ( + math.radians(SWAY_A_PITCH_DEG) + * loud + * env + * math.sin(2 * math.pi * SWAY_F_PITCH * self.t + self.phase_pitch) + ) + yaw = ( + math.radians(SWAY_A_YAW_DEG) + * loud + * env + * math.sin(2 * math.pi * SWAY_F_YAW * self.t + self.phase_yaw) + ) + roll = ( + math.radians(SWAY_A_ROLL_DEG) + * loud + * env + * math.sin(2 * math.pi * SWAY_F_ROLL * self.t + self.phase_roll) + ) + x_mm = SWAY_A_X_MM * loud * env * math.sin(2 * math.pi * SWAY_F_X * self.t + self.phase_x) + y_mm = SWAY_A_Y_MM * loud * env * math.sin(2 * math.pi * SWAY_F_Y * self.t + self.phase_y) + z_mm = SWAY_A_Z_MM * loud * env * math.sin(2 * math.pi * SWAY_F_Z * self.t + self.phase_z) + + out.append( + { + "pitch_rad": pitch, + "yaw_rad": yaw, + "roll_rad": roll, + "pitch_deg": math.degrees(pitch), + "yaw_deg": math.degrees(yaw), + "roll_deg": math.degrees(roll), + "x_mm": x_mm, + "y_mm": y_mm, + "z_mm": z_mm, + }, + ) + + return out diff --git a/src/reachy_mini_receptionist/calendar_data.py b/src/reachy_mini_receptionist/calendar_data.py new file mode 100644 index 0000000000000000000000000000000000000000..25577e13500bfa64a321fc5abf21c5987bb73b4a --- /dev/null +++ b/src/reachy_mini_receptionist/calendar_data.py @@ -0,0 +1,139 @@ +"""Calendar data — appointments source for the receptionist. + +Single source: a Google Calendar (or any iCal feed) configured via the +``RECEPTION_ICS_URL`` env var. Operators add events in Google Calendar; +the robot fetches every ~5 min via ``ical_calendar.py``. Title +convention: ``" with "`` (host resolved through +``employees.py``). + +When ``RECEPTION_ICS_URL`` is unset OR the feed is unreachable, this +module returns an EMPTY calendar. That's intentional — the receptionist +supports exactly two visitor paths: + +1. **Scheduled visitor** — appointment exists in the iCal feed; bot + matches by visitor name and emails the host. +2. **Walk-in to see an employee** — visitor names a host that lives in + the SQLite Employee directory (managed from the dashboard's + Employees panel); bot calls ``lookup_employee`` and emails the host. + +There is intentionally no hardcoded demo schedule fallback. If you want +demo data, add it to Google Calendar; if you want a host-only flow, add +the host via the dashboard. + +The ``visiting`` field on each returned appointment is always an email +address (resolved through the employee directory). If a host can't be +resolved, the original string is preserved so the LLM can flag it. +""" +from __future__ import annotations + +import asyncio +import os +from datetime import datetime +from typing import Any, Dict, List, Optional + +from reachy_mini_receptionist import employees + + +def _resolve_visiting(visiting: str) -> str: + """Return the email for a `visiting` reference, falling back to itself. + + Looks up via the employee directory first; if `visiting` already contains + `@` (one-off external host), returns it unchanged. + """ + if not visiting: + return "" + if "@" in visiting: + return visiting + email = employees.find_email_for(visiting) + return email or visiting + + +def _appointments_from_ical(ics_url: str) -> List[Dict[str, Any]]: + """Pull today's appointments from an iCal feed and reshape them. + + Drops the iCal-specific helper fields (``_host_query``, ``_dt``) and + resolves the host name to an email through the employee directory. + """ + from reachy_mini_receptionist import ical_calendar + + raw = ical_calendar.fetch_appointments(ics_url) + out: List[Dict[str, Any]] = [] + for ev in raw: + host_query = ev.get("_host_query", "") + out.append({ + "time": ev.get("time", ""), + "name": ev.get("name", ""), + "note": ev.get("note", ""), + "visiting": _resolve_visiting(host_query) if host_query else "", + }) + return out + + +def get_appointments() -> List[dict]: + """Return today's appointment list with ``visiting`` resolved to an email. + + Pulls live from the iCal feed when ``RECEPTION_ICS_URL`` is set + (cached ~5 min). Returns an empty list otherwise — walk-in flow + (via ``lookup_employee``) handles visitors who aren't on the schedule. + + Each item has: + time (str) — e.g. "11:00 AM" + name (str) — guest name + note (str) — short description (may be empty) + visiting (str) — host email (resolved from employee directory) + """ + ics_url = (os.getenv("RECEPTION_ICS_URL") or "").strip() + if not ics_url: + return [] + return _appointments_from_ical(ics_url) + + +async def get_appointments_async() -> List[dict]: + """Async wrapper for ``get_appointments`` — offloads the sync iCal HTTP + fetch to a worker thread so async callers (realtime audio loop, tool + completion handlers) don't block the event loop on a 10-second HTTP + timeout. Cache hits return immediately; only the underlying httpx.get + call gets thread-offloaded. + """ + return await asyncio.to_thread(get_appointments) + + +def format_for_llm() -> str: + """Return a human-readable calendar string for the LLM.""" + today = datetime.now().strftime("%A, %B %d %Y") + appts = get_appointments() + if not appts: + return ( + f"Today is {today}. No scheduled appointments in the calendar. " + "Walk-in visitors should be routed via lookup_employee." + ) + lines = [f"Today is {today}. Appointments:"] + for appt in appts: + lines.append(f" {appt['time']}: {appt['name']} — {appt['note']}") + return "\n".join(lines) + + +def get_appointment_for_name(name: str) -> Optional[dict]: + """Find an appointment by guest name (case-insensitive).""" + target = (name or "").strip().lower() + for appt in get_appointments(): + if appt["name"].lower() == target: + return appt + return None + + +def find_appointment_for_employee(employee_query: str) -> Optional[dict]: + """Find today's appointment whose host matches ``employee_query``. + + Resolves the query through the employee directory first so callers can + pass a name OR alias OR email. Returns the first matching appointment, + or None if nothing on today's schedule is for that host. + """ + target_email = employees.find_email_for(employee_query) or employee_query + target_email = (target_email or "").strip().lower() + if not target_email: + return None + for appt in get_appointments(): + if (appt.get("visiting") or "").strip().lower() == target_email: + return appt + return None diff --git a/src/reachy_mini_receptionist/camera_worker.py b/src/reachy_mini_receptionist/camera_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..e625c00936d73edbe6c9bcf677f11d71a3409d89 --- /dev/null +++ b/src/reachy_mini_receptionist/camera_worker.py @@ -0,0 +1,241 @@ +"""Camera worker thread with frame buffering and face tracking. + +Ported from main_works.py camera_worker() function to provide: +- 30Hz+ camera polling with thread-safe frame buffering +- Face tracking integration with smooth interpolation +- Latest frame always available for tools +""" + +import time +import logging +import threading +from typing import Any, List, Tuple + +import numpy as np +from numpy.typing import NDArray +from scipy.spatial.transform import Rotation as R + +from reachy_mini import ReachyMini +from reachy_mini.utils.interpolation import linear_pose_interpolation + + +logger = logging.getLogger(__name__) + + +class CameraWorker: + """Thread-safe camera worker with frame buffering and face tracking.""" + + def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None: + """Initialize.""" + self.reachy_mini = reachy_mini + self.head_tracker = head_tracker + + # Thread-safe frame storage + self.latest_frame: NDArray[np.uint8] | None = None + self.frame_lock = threading.Lock() + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + + # Face tracking state + self.is_head_tracking_enabled = True + self.face_tracking_offsets: List[float] = [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ] # x, y, z, roll, pitch, yaw + self.face_tracking_lock = threading.Lock() + + # Face tracking timing variables (same as main_works.py) + self.last_face_detected_time: float | None = None + self.interpolation_start_time: float | None = None + self.interpolation_start_pose: NDArray[np.float32] | None = None + self.face_lost_delay = 2.0 # seconds to wait before starting interpolation + self.interpolation_duration = 1.0 # seconds to interpolate back to neutral + + # Track state changes + self.previous_head_tracking_state = self.is_head_tracking_enabled + + def get_latest_frame(self) -> NDArray[np.uint8] | None: + """Get the latest frame (thread-safe).""" + with self.frame_lock: + if self.latest_frame is None: + return None + # Return a copy in original BGR format (OpenCV native) + return self.latest_frame.copy() + + def get_face_tracking_offsets( + self, + ) -> Tuple[float, float, float, float, float, float]: + """Get current face tracking offsets (thread-safe).""" + with self.face_tracking_lock: + offsets = self.face_tracking_offsets + return (offsets[0], offsets[1], offsets[2], offsets[3], offsets[4], offsets[5]) + + def set_head_tracking_enabled(self, enabled: bool) -> None: + """Enable/disable head tracking.""" + self.is_head_tracking_enabled = enabled + logger.info(f"Head tracking {'enabled' if enabled else 'disabled'}") + + def start(self) -> None: + """Start the camera worker loop in a thread.""" + self._stop_event.clear() + self._thread = threading.Thread(target=self.working_loop, daemon=True) + self._thread.start() + logger.debug("Camera worker started") + + def stop(self) -> None: + """Stop the camera worker loop.""" + self._stop_event.set() + if self._thread is not None: + self._thread.join() + + logger.debug("Camera worker stopped") + + def working_loop(self) -> None: + """Enable the camera worker loop. + + Ported from main_works.py camera_worker() with same logic. + """ + logger.debug("Starting camera working loop") + + # Initialize head tracker if available + neutral_pose = np.eye(4) # Neutral pose (identity matrix) + self.previous_head_tracking_state = self.is_head_tracking_enabled + + while not self._stop_event.is_set(): + try: + current_time = time.time() + + # Get frame from robot + frame = self.reachy_mini.media.get_frame() + + if frame is not None: + # Thread-safe frame storage + with self.frame_lock: + self.latest_frame = frame # .copy() + + # Check if face tracking was just disabled + if self.previous_head_tracking_state and not self.is_head_tracking_enabled: + # Face tracking was just disabled - start interpolation to neutral + self.last_face_detected_time = current_time # Trigger the face-lost logic + self.interpolation_start_time = None # Will be set by the face-lost interpolation + self.interpolation_start_pose = None + + # Update tracking state + self.previous_head_tracking_state = self.is_head_tracking_enabled + + # Handle face tracking if enabled and head tracker available + if self.is_head_tracking_enabled and self.head_tracker is not None: + eye_center, _ = self.head_tracker.get_head_position(frame) + + if eye_center is not None: + # Face detected - immediately switch to tracking + self.last_face_detected_time = current_time + self.interpolation_start_time = None # Stop any interpolation + + # Convert normalized coordinates to pixel coordinates + h, w, _ = frame.shape + eye_center_norm = (eye_center + 1) / 2 + eye_center_pixels = [ + eye_center_norm[0] * w, + eye_center_norm[1] * h, + ] + + # Get the head pose needed to look at the target, but don't perform movement + target_pose = self.reachy_mini.look_at_image( + eye_center_pixels[0], + eye_center_pixels[1], + duration=0.0, + perform_movement=False, + ) + + # Extract translation and rotation from the target pose directly + translation = target_pose[:3, 3] + rotation = R.from_matrix(target_pose[:3, :3]).as_euler("xyz", degrees=False) + + # Scale down translation and rotation because smaller FOV + translation *= 0.6 + rotation *= 0.6 + + # Thread-safe update of face tracking offsets (use pose as-is) + with self.face_tracking_lock: + self.face_tracking_offsets = [ + translation[0], + translation[1], + translation[2], # x, y, z + rotation[0], + rotation[1], + rotation[2], # roll, pitch, yaw + ] + + # No face detected while tracking enabled - set face lost timestamp + elif self.last_face_detected_time is None or self.last_face_detected_time == current_time: + # Only update if we haven't already set a face lost time + # (current_time check prevents overriding the disable-triggered timestamp) + pass + + # Handle smooth interpolation (works for both face-lost and tracking-disabled cases) + if self.last_face_detected_time is not None: + time_since_face_lost = current_time - self.last_face_detected_time + + if time_since_face_lost >= self.face_lost_delay: + # Start interpolation if not already started + if self.interpolation_start_time is None: + self.interpolation_start_time = current_time + # Capture current pose as start of interpolation + with self.face_tracking_lock: + current_translation = self.face_tracking_offsets[:3] + current_rotation_euler = self.face_tracking_offsets[3:] + # Convert to 4x4 pose matrix + pose_matrix = np.eye(4, dtype=np.float32) + pose_matrix[:3, 3] = current_translation + pose_matrix[:3, :3] = R.from_euler( + "xyz", + current_rotation_euler, + ).as_matrix() + self.interpolation_start_pose = pose_matrix + + # Calculate interpolation progress (t from 0 to 1) + elapsed_interpolation = current_time - self.interpolation_start_time + t = min(1.0, elapsed_interpolation / self.interpolation_duration) + + # Interpolate between current pose and neutral pose + interpolated_pose = linear_pose_interpolation( + self.interpolation_start_pose, + neutral_pose, + t, + ) + + # Extract translation and rotation from interpolated pose + translation = interpolated_pose[:3, 3] + rotation = R.from_matrix(interpolated_pose[:3, :3]).as_euler("xyz", degrees=False) + + # Thread-safe update of face tracking offsets + with self.face_tracking_lock: + self.face_tracking_offsets = [ + translation[0], + translation[1], + translation[2], # x, y, z + rotation[0], + rotation[1], + rotation[2], # roll, pitch, yaw + ] + + # If interpolation is complete, reset timing + if t >= 1.0: + self.last_face_detected_time = None + self.interpolation_start_time = None + self.interpolation_start_pose = None + # else: Keep current offsets (within 2s delay period) + + # Small sleep to prevent excessive CPU usage (same as main_works.py) + time.sleep(0.04) + + except Exception as e: + logger.error(f"Camera worker error: {e}") + time.sleep(0.1) # Longer sleep on error + + logger.debug("Camera worker thread exited") diff --git a/src/reachy_mini_receptionist/config.py b/src/reachy_mini_receptionist/config.py new file mode 100644 index 0000000000000000000000000000000000000000..12e8c64df3517d9bd47462812561bb3cde602b5d --- /dev/null +++ b/src/reachy_mini_receptionist/config.py @@ -0,0 +1,217 @@ +import os +import sys +import logging +from pathlib import Path + +from dotenv import find_dotenv, load_dotenv + + +# Locked profile: set to a profile name (e.g., "astronomer") to lock the app +# to that profile and disable all profile switching. Leave as None for normal behavior. +LOCKED_PROFILE: str | None = "_reachy_mini_receptionist_locked_profile" +DEFAULT_PROFILES_DIRECTORY = Path(__file__).parent / "profiles" + +logger = logging.getLogger(__name__) + + +def _env_flag(name: str, default: bool = False) -> bool: + """Parse a boolean environment flag. + + Accepted truthy values: 1, true, yes, on + Accepted falsy values: 0, false, no, off + """ + raw = os.getenv(name) + if raw is None: + return default + + value = raw.strip().lower() + if value in {"1", "true", "yes", "on"}: + return True + if value in {"0", "false", "no", "off"}: + return False + + logger.warning("Invalid boolean value for %s=%r, using default=%s", name, raw, default) + return default + + +def _collect_profile_names(profiles_root: Path) -> set[str]: + """Return profile folder names from a profiles root directory.""" + if not profiles_root.exists() or not profiles_root.is_dir(): + return set() + return {p.name for p in profiles_root.iterdir() if p.is_dir()} + + +def _collect_tool_module_names(tools_root: Path) -> set[str]: + """Return tool module names from a tools directory.""" + if not tools_root.exists() or not tools_root.is_dir(): + return set() + ignored = {"__init__", "core_tools"} + return { + p.stem + for p in tools_root.glob("*.py") + if p.is_file() and p.stem not in ignored + } + + +def _raise_on_name_collisions( + *, + label: str, + external_root: Path, + internal_root: Path, + external_names: set[str], + internal_names: set[str], +) -> None: + """Raise with a clear message when external/internal names collide.""" + collisions = sorted(external_names & internal_names) + if not collisions: + return + + raise RuntimeError( + f"Config.__init__(): Ambiguous {label} names found in both external and built-in libraries: {collisions}. " + f"External {label} root: {external_root}. Built-in {label} root: {internal_root}. " + f"Please rename the conflicting external {label}(s) to continue." + ) + + +# Validate LOCKED_PROFILE at startup +if LOCKED_PROFILE is not None: + _profiles_dir = DEFAULT_PROFILES_DIRECTORY + _profile_path = _profiles_dir / LOCKED_PROFILE + _instructions_file = _profile_path / "instructions.txt" + if not _profile_path.is_dir(): + print(f"Error: LOCKED_PROFILE '{LOCKED_PROFILE}' does not exist in {_profiles_dir}", file=sys.stderr) + sys.exit(1) + if not _instructions_file.is_file(): + print(f"Error: LOCKED_PROFILE '{LOCKED_PROFILE}' has no instructions.txt", file=sys.stderr) + sys.exit(1) + +_skip_dotenv = _env_flag("REACHY_MINI_SKIP_DOTENV", default=False) + +if _skip_dotenv: + logger.info("Skipping .env loading because REACHY_MINI_SKIP_DOTENV is set") +else: + # Locate .env file (search upward from current working directory) + dotenv_path = find_dotenv(usecwd=True) + + if dotenv_path: + # Load .env and override environment variables + load_dotenv(dotenv_path=dotenv_path, override=True) + logger.info(f"Configuration loaded from {dotenv_path}") + else: + logger.warning("No .env file found, using environment variables") + + +class Config: + """Configuration class for the receptionist app.""" + + # Required + OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") # The key is downloaded in console.py if needed + + # Optional + MODEL_NAME = os.getenv("MODEL_NAME", "gpt-realtime") + HF_HOME = os.getenv("HF_HOME", "./cache") + LOCAL_VISION_MODEL = os.getenv("LOCAL_VISION_MODEL", "HuggingFaceTB/SmolVLM2-2.2B-Instruct") + HF_TOKEN = os.getenv("HF_TOKEN") # Optional, falls back to hf auth login if not set + + logger.debug(f"Model: {MODEL_NAME}, HF_HOME: {HF_HOME}, Vision Model: {LOCAL_VISION_MODEL}") + + _profiles_directory_env = os.getenv("REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY") + PROFILES_DIRECTORY = ( + Path(_profiles_directory_env) if _profiles_directory_env else Path(__file__).parent / "profiles" + ) + _tools_directory_env = os.getenv("REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY") + TOOLS_DIRECTORY = Path(_tools_directory_env) if _tools_directory_env else None + AUTOLOAD_EXTERNAL_TOOLS = _env_flag("AUTOLOAD_EXTERNAL_TOOLS", default=False) + REACHY_MINI_CUSTOM_PROFILE = LOCKED_PROFILE or os.getenv("REACHY_MINI_CUSTOM_PROFILE") + + logger.debug(f"Custom Profile: {REACHY_MINI_CUSTOM_PROFILE}") + + def __init__(self) -> None: + """Initialize the configuration.""" + if self.REACHY_MINI_CUSTOM_PROFILE and self.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + selected_profile_path = self.PROFILES_DIRECTORY / self.REACHY_MINI_CUSTOM_PROFILE + if not selected_profile_path.is_dir(): + available_profiles = sorted(_collect_profile_names(self.PROFILES_DIRECTORY)) + raise RuntimeError( + "Config.__init__(): Selected profile " + f"'{self.REACHY_MINI_CUSTOM_PROFILE}' was not found in external profiles root " + f"{self.PROFILES_DIRECTORY}. " + f"Available external profiles: {available_profiles}. " + "Either set 'REACHY_MINI_CUSTOM_PROFILE' to one of the available external profiles " + "or unset 'REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY' to use built-in profiles." + ) + + if self.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + external_profiles = _collect_profile_names(self.PROFILES_DIRECTORY) + internal_profiles = _collect_profile_names(DEFAULT_PROFILES_DIRECTORY) + _raise_on_name_collisions( + label="profile", + external_root=self.PROFILES_DIRECTORY, + internal_root=DEFAULT_PROFILES_DIRECTORY, + external_names=external_profiles, + internal_names=internal_profiles, + ) + + if self.TOOLS_DIRECTORY is not None: + builtin_tools_root = Path(__file__).parent / "tools" + external_tools = _collect_tool_module_names(self.TOOLS_DIRECTORY) + internal_tools = _collect_tool_module_names(builtin_tools_root) + _raise_on_name_collisions( + label="tool", + external_root=self.TOOLS_DIRECTORY, + internal_root=builtin_tools_root, + external_names=external_tools, + internal_names=internal_tools, + ) + + if self.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + logger.warning( + "Environment variable 'REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY' is set. " + "Profiles (instructions.txt, ...) will be loaded from %s.", + self.PROFILES_DIRECTORY, + ) + else: + logger.info( + "'REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY' is not set. " + "Using built-in profiles from %s.", + DEFAULT_PROFILES_DIRECTORY, + ) + + if self.TOOLS_DIRECTORY is not None: + logger.warning( + "Environment variable 'REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY' is set. " + "External tools will be loaded from %s.", + self.TOOLS_DIRECTORY, + ) + else: + logger.info( + "'REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY' is not set. " + "Using built-in shared tools only." + ) + + +config = Config() + + +def set_custom_profile(profile: str | None) -> None: + """Update the selected custom profile at runtime and expose it via env. + + This ensures modules that read `config` and code that inspects the + environment see a consistent value. + """ + if LOCKED_PROFILE is not None: + return + try: + config.REACHY_MINI_CUSTOM_PROFILE = profile + except Exception: + pass + try: + import os as _os + + if profile: + _os.environ["REACHY_MINI_CUSTOM_PROFILE"] = profile + else: + # Remove to reflect default + _os.environ.pop("REACHY_MINI_CUSTOM_PROFILE", None) + except Exception: + pass diff --git a/src/reachy_mini_receptionist/console.py b/src/reachy_mini_receptionist/console.py new file mode 100644 index 0000000000000000000000000000000000000000..a2b7f3cfcdff9e0321b43408ac04285219090874 --- /dev/null +++ b/src/reachy_mini_receptionist/console.py @@ -0,0 +1,527 @@ +"""Bidirectional local audio stream with optional settings UI. + +In headless mode, there is no Gradio UI. If the OpenAI API key is not +available via environment/.env, we expose a minimal settings page via the +Reachy Mini Apps settings server to let non-technical users enter it. + +The settings UI is served from this package's ``static/`` folder and offers a +single password field to set ``OPENAI_API_KEY``. Once set, we persist it to the +app instance's ``.env`` file (if available) and proceed to start streaming. +""" + +import os +import sys +import time +import asyncio +import logging +from typing import List, Optional +from pathlib import Path + +from fastrtc import AdditionalOutputs, audio_to_float32 +from scipy.signal import resample + +from reachy_mini import ReachyMini +from reachy_mini.media.media_manager import MediaBackend +from reachy_mini_receptionist.config import LOCKED_PROFILE, config +from reachy_mini_receptionist.openai_realtime import OpenaiRealtimeHandler +from reachy_mini_receptionist.headless_personality_ui import mount_personality_routes + + +try: + # FastAPI is provided by the Reachy Mini Apps runtime + from fastapi import FastAPI, Response + from pydantic import BaseModel + from fastapi.responses import FileResponse, JSONResponse, RedirectResponse + from starlette.staticfiles import StaticFiles +except Exception: # pragma: no cover - only loaded when settings_app is used + FastAPI = object # type: ignore + FileResponse = object # type: ignore + JSONResponse = object # type: ignore + StaticFiles = object # type: ignore + BaseModel = object # type: ignore + + +logger = logging.getLogger(__name__) + + +class LocalStream: + """LocalStream using Reachy Mini's recorder/player.""" + + def __init__( + self, + handler: OpenaiRealtimeHandler, + robot: ReachyMini, + *, + settings_app: Optional[FastAPI] = None, + instance_path: Optional[str] = None, + ): + """Initialize the stream with an OpenAI realtime handler and pipelines. + + - ``settings_app``: the Reachy Mini Apps FastAPI to attach settings endpoints. + - ``instance_path``: directory where per-instance ``.env`` should be stored. + """ + self.handler = handler + self._robot = robot + self._stop_event = asyncio.Event() + self._tasks: List[asyncio.Task[None]] = [] + # Allow the handler to flush the player queue when appropriate. + self.handler._clear_queue = self.clear_audio_queue + self._settings_app: Optional[FastAPI] = settings_app + self._instance_path: Optional[str] = instance_path + self._settings_initialized = False + self._asyncio_loop = None + + # ---- Settings UI (only when API key is missing) ---- + def _read_env_lines(self, env_path: Path) -> list[str]: + """Load env file contents or a template as a list of lines.""" + inst = env_path.parent + try: + if env_path.exists(): + try: + return env_path.read_text(encoding="utf-8").splitlines() + except Exception: + return [] + template_text = None + ex = inst / ".env.example" + if ex.exists(): + try: + template_text = ex.read_text(encoding="utf-8") + except Exception: + template_text = None + if template_text is None: + try: + cwd_example = Path.cwd() / ".env.example" + if cwd_example.exists(): + template_text = cwd_example.read_text(encoding="utf-8") + except Exception: + template_text = None + if template_text is None: + packaged = Path(__file__).parent / ".env.example" + if packaged.exists(): + try: + template_text = packaged.read_text(encoding="utf-8") + except Exception: + template_text = None + return template_text.splitlines() if template_text else [] + except Exception: + return [] + + def _persist_api_key(self, key: str) -> None: + """Persist API key to environment and instance ``.env`` if possible. + + Behavior: + - Always sets ``OPENAI_API_KEY`` in process env and in-memory config. + - Writes/updates ``/.env``: + * If ``.env`` exists, replaces/append OPENAI_API_KEY line. + * Else, copies template from ``/.env.example`` when present, + otherwise falls back to the packaged template + ``reachy_mini_receptionist/.env.example``. + * Ensures the resulting file contains the full template plus the key. + - Loads the written ``.env`` into the current process environment. + """ + k = (key or "").strip() + if not k: + return + # Update live process env and config so consumers see it immediately + try: + os.environ["OPENAI_API_KEY"] = k + except Exception: # best-effort + pass + try: + config.OPENAI_API_KEY = k + except Exception: + pass + + if not self._instance_path: + return + try: + inst = Path(self._instance_path) + env_path = inst / ".env" + lines = self._read_env_lines(env_path) + replaced = False + for i, ln in enumerate(lines): + if ln.strip().startswith("OPENAI_API_KEY="): + lines[i] = f"OPENAI_API_KEY={k}" + replaced = True + break + if not replaced: + lines.append(f"OPENAI_API_KEY={k}") + final_text = "\n".join(lines) + "\n" + env_path.write_text(final_text, encoding="utf-8") + logger.info("Persisted OPENAI_API_KEY to %s", env_path) + + # Load the newly written .env into this process to ensure downstream imports see it + try: + from dotenv import load_dotenv + + load_dotenv(dotenv_path=str(env_path), override=True) + except Exception: + pass + except Exception as e: + logger.warning("Failed to persist OPENAI_API_KEY: %s", e) + + def _persist_personality(self, profile: Optional[str]) -> None: + """Persist the startup personality to the instance .env and config.""" + if LOCKED_PROFILE is not None: + return + selection = (profile or "").strip() or None + try: + from reachy_mini_receptionist.config import set_custom_profile + + set_custom_profile(selection) + except Exception: + pass + + if not self._instance_path: + return + try: + env_path = Path(self._instance_path) / ".env" + lines = self._read_env_lines(env_path) + replaced = False + for i, ln in enumerate(list(lines)): + if ln.strip().startswith("REACHY_MINI_CUSTOM_PROFILE="): + if selection: + lines[i] = f"REACHY_MINI_CUSTOM_PROFILE={selection}" + else: + lines.pop(i) + replaced = True + break + if selection and not replaced: + lines.append(f"REACHY_MINI_CUSTOM_PROFILE={selection}") + if selection is None and not env_path.exists(): + return + final_text = "\n".join(lines) + "\n" + env_path.write_text(final_text, encoding="utf-8") + logger.info("Persisted startup personality to %s", env_path) + try: + from dotenv import load_dotenv + + load_dotenv(dotenv_path=str(env_path), override=True) + except Exception: + pass + except Exception as e: + logger.warning("Failed to persist REACHY_MINI_CUSTOM_PROFILE: %s", e) + + def _read_persisted_personality(self) -> Optional[str]: + """Read persisted startup personality from instance .env (if any).""" + if not self._instance_path: + return None + env_path = Path(self._instance_path) / ".env" + try: + if env_path.exists(): + for ln in env_path.read_text(encoding="utf-8").splitlines(): + if ln.strip().startswith("REACHY_MINI_CUSTOM_PROFILE="): + _, _, val = ln.partition("=") + v = val.strip() + return v or None + except Exception: + pass + return None + + def _init_settings_ui_if_needed(self) -> None: + """Attach minimal settings UI to the settings app. + + Always mounts the UI when a settings_app is provided so that users + see a confirmation message even if the API key is already configured. + """ + if self._settings_initialized: + return + if self._settings_app is None: + return + + static_dir = Path(__file__).parent / "static" + index_file = static_dir / "index.html" + + if hasattr(self._settings_app, "mount"): + try: + # Serve /static/* assets + self._settings_app.mount("/static", StaticFiles(directory=str(static_dir)), name="static") + except Exception: + pass + + class ApiKeyPayload(BaseModel): + openai_api_key: str + + # GET / -> redirect to /dashboard (the receptionist control room) + @self._settings_app.get("/") + def _root() -> RedirectResponse: + return RedirectResponse(url="/dashboard") + + # GET /favicon.ico -> optional, avoid noisy 404s on some browsers + @self._settings_app.get("/favicon.ico") + def _favicon() -> Response: + return Response(status_code=204) + + # GET /status -> whether key is set + @self._settings_app.get("/status") + def _status() -> JSONResponse: + has_key = bool(config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip()) + return JSONResponse({"has_key": has_key}) + + # GET /ready -> whether backend finished loading tools + @self._settings_app.get("/ready") + def _ready() -> JSONResponse: + try: + mod = sys.modules.get("reachy_mini_receptionist.tools.core_tools") + ready = bool(getattr(mod, "_TOOLS_INITIALIZED", False)) if mod else False + except Exception: + ready = False + return JSONResponse({"ready": ready}) + + # POST /openai_api_key -> set/persist key + @self._settings_app.post("/openai_api_key") + def _set_key(payload: ApiKeyPayload) -> JSONResponse: + key = (payload.openai_api_key or "").strip() + if not key: + return JSONResponse({"ok": False, "error": "empty_key"}, status_code=400) + self._persist_api_key(key) + return JSONResponse({"ok": True}) + + # POST /validate_api_key -> validate key without persisting it + @self._settings_app.post("/validate_api_key") + async def _validate_key(payload: ApiKeyPayload) -> JSONResponse: + key = (payload.openai_api_key or "").strip() + if not key: + return JSONResponse({"valid": False, "error": "empty_key"}, status_code=400) + + # Try to validate by checking if we can fetch the models + try: + import httpx + + headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"} + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get("https://api.openai.com/v1/models", headers=headers) + if response.status_code == 200: + return JSONResponse({"valid": True}) + elif response.status_code == 401: + return JSONResponse({"valid": False, "error": "invalid_api_key"}, status_code=401) + else: + return JSONResponse( + {"valid": False, "error": "validation_failed"}, status_code=response.status_code + ) + except Exception as e: + logger.warning(f"API key validation failed: {e}") + return JSONResponse({"valid": False, "error": "validation_error"}, status_code=500) + + self._settings_initialized = True + + def launch(self) -> None: + """Start the recorder/player and run the async processing loops. + + If the OpenAI key is missing, expose a tiny settings UI via the + Reachy Mini settings server to collect it before starting streams. + """ + self._stop_event.clear() + + # Try to load an existing instance .env first (covers subsequent runs) + if self._instance_path: + try: + from dotenv import load_dotenv + + from reachy_mini_receptionist.config import set_custom_profile + + env_path = Path(self._instance_path) / ".env" + if env_path.exists(): + load_dotenv(dotenv_path=str(env_path), override=True) + # Update config with newly loaded values + new_key = os.getenv("OPENAI_API_KEY", "").strip() + if new_key: + try: + config.OPENAI_API_KEY = new_key + except Exception: + pass + if LOCKED_PROFILE is None: + new_profile = os.getenv("REACHY_MINI_CUSTOM_PROFILE") + if new_profile is not None: + try: + set_custom_profile(new_profile.strip() or None) + except Exception: + pass # Best-effort profile update + except Exception: + pass # Instance .env loading is optional; continue with defaults + + # If key is still missing, try to download one from HuggingFace + if not (config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip()): + logger.info("OPENAI_API_KEY not set, attempting to download from HuggingFace...") + try: + from gradio_client import Client + client = Client("HuggingFaceM4/gradium_setup", verbose=False) + key, status = client.predict(api_name="/claim_b_key") + if key and key.strip(): + logger.info("Successfully downloaded API key from HuggingFace") + # Persist it immediately + self._persist_api_key(key) + except Exception as e: + logger.warning(f"Failed to download API key from HuggingFace: {e}") + + # Always expose settings UI if a settings app is available + # (do this AFTER loading/downloading the key so status endpoint sees the right value) + self._init_settings_ui_if_needed() + + # If key is still missing -> wait until provided via the settings UI + if not (config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip()): + logger.warning("OPENAI_API_KEY not found. Open the app settings page to enter it.") + # Poll until the key becomes available (set via the settings UI) + try: + while not (config.OPENAI_API_KEY and str(config.OPENAI_API_KEY).strip()): + time.sleep(0.2) + except KeyboardInterrupt: + logger.info("Interrupted while waiting for API key.") + return + + # Start media after key is set/available + self._robot.media.start_recording() + self._robot.media.start_playing() + time.sleep(1) # give some time to the pipelines to start + + async def runner() -> None: + # Capture loop for cross-thread personality actions + loop = asyncio.get_running_loop() + self._asyncio_loop = loop # type: ignore[assignment] + # Mount personality routes now that loop and handler are available + try: + if self._settings_app is not None: + mount_personality_routes( + self._settings_app, + self.handler, + lambda: self._asyncio_loop, + persist_personality=self._persist_personality, + get_persisted_personality=self._read_persisted_personality, + ) + except Exception: + pass + self._tasks = [ + asyncio.create_task(self.handler.start_up(), name="openai-handler"), + asyncio.create_task(self.record_loop(), name="stream-record-loop"), + asyncio.create_task(self.play_loop(), name="stream-play-loop"), + ] + try: + await asyncio.gather(*self._tasks) + except asyncio.CancelledError: + logger.info("Tasks cancelled during shutdown") + finally: + # Ensure handler connection is closed + await self.handler.shutdown() + + asyncio.run(runner()) + + def close(self) -> None: + """Stop the stream and underlying media pipelines. + + This method: + - Stops audio recording and playback first + - Sets the stop event to signal async loops to terminate + - Cancels all pending async tasks (openai-handler, record-loop, play-loop) + """ + logger.info("Stopping LocalStream...") + + # Stop media pipelines FIRST before cancelling async tasks + # This ensures clean shutdown before PortAudio cleanup + try: + self._robot.media.stop_recording() + except Exception as e: + logger.debug(f"Error stopping recording (may already be stopped): {e}") + + try: + self._robot.media.stop_playing() + except Exception as e: + logger.debug(f"Error stopping playback (may already be stopped): {e}") + + # Now signal async loops to stop + self._stop_event.set() + + # Cancel all running tasks + for task in self._tasks: + if not task.done(): + task.cancel() + + def clear_audio_queue(self) -> None: + """Flush the player's appsrc to drop any queued audio immediately.""" + logger.info("User intervention: flushing player queue") + if self._robot.media.backend == MediaBackend.GSTREAMER: + # Directly flush gstreamer audio pipe + self._robot.media.audio.clear_player() + elif self._robot.media.backend == MediaBackend.DEFAULT or self._robot.media.backend == MediaBackend.DEFAULT_NO_VIDEO: + self._robot.media.audio.clear_output_buffer() + self.handler.output_queue = asyncio.Queue() + + async def record_loop(self) -> None: + """Read mic frames from the recorder and forward them to the handler.""" + input_sample_rate = self._robot.media.get_input_audio_samplerate() + logger.debug(f"Audio recording started at {input_sample_rate} Hz") + + while not self._stop_event.is_set(): + audio_frame = self._robot.media.get_audio_sample() + if audio_frame is not None: + await self.handler.receive((input_sample_rate, audio_frame)) + await asyncio.sleep(0) # avoid busy loop + + async def play_loop(self) -> None: + """Fetch outputs from the handler: log text and play audio frames.""" + while not self._stop_event.is_set(): + handler_output = await self.handler.emit() + + if isinstance(handler_output, AdditionalOutputs): + for msg in handler_output.args: + content = msg.get("content", "") + if isinstance(content, str): + logger.info( + "role=%s content=%s", + msg.get("role"), + content if len(content) < 500 else content[:500] + "…", + ) + + elif isinstance(handler_output, tuple): + input_sample_rate, audio_data = handler_output + output_sample_rate = self._robot.media.get_output_audio_samplerate() + + # Reshape if needed + if audio_data.ndim == 2: + # Scipy channels last convention + if audio_data.shape[1] > audio_data.shape[0]: + audio_data = audio_data.T + # Multiple channels -> Mono channel + if audio_data.shape[1] > 1: + audio_data = audio_data[:, 0] + + # Cast if needed + audio_frame = audio_to_float32(audio_data) + + # Drop empty / sub-sample chunks. Some Gemini Live preview + # models (e.g. gemini-3.1-flash-live-preview as of + # 2026-05-21) emit 2-byte placeholder chunks. Without + # this guard, scipy.signal.resample below does + # `len_in / len_out` and crashes with ZeroDivisionError + # when the resampled target length rounds to 0, + # killing the whole console play_loop and the app with + # it. Skipping is the safe behaviour — a truly empty + # chunk has nothing to play anyway. + if audio_frame.size == 0 or len(audio_frame) < 2: + logger.debug( + "play_loop: skipping near-empty audio frame " + "(len=%d, input_sr=%s, output_sr=%s)", + len(audio_frame), input_sample_rate, output_sample_rate, + ) + await asyncio.sleep(0) + continue + + # Resample if needed + if input_sample_rate != output_sample_rate: + target_len = int(len(audio_frame) * output_sample_rate / input_sample_rate) + if target_len < 1: + # Resample would divide by zero — skip rather than crash. + logger.debug( + "play_loop: skipping frame that would resample to 0 " + "samples (len=%d, %s->%s)", + len(audio_frame), input_sample_rate, output_sample_rate, + ) + await asyncio.sleep(0) + continue + audio_frame = resample(audio_frame, target_len) + + self._robot.media.push_audio_sample(audio_frame) + + else: + logger.debug("Ignoring output type=%s", type(handler_output).__name__) + + await asyncio.sleep(0) # yield to event loop diff --git a/src/reachy_mini_receptionist/conversation_controller.py b/src/reachy_mini_receptionist/conversation_controller.py new file mode 100644 index 0000000000000000000000000000000000000000..b7d3bb476cdcb92b90ecb317691aaf4864b9d148 --- /dev/null +++ b/src/reachy_mini_receptionist/conversation_controller.py @@ -0,0 +1,586 @@ +"""Conversation controller — translates events into session state transitions. + +This is the workflow engine. It listens for: +- Face state events from FaceRecognitionWorker. +- Tool call completions from the realtime handler. + +And decides which ReceptionState transition should fire on the SessionManager. + +Also exposes ``next_action_hint(state)`` — short directives the realtime +handler appends to its session context push so the LLM gets per-state +workflow guidance dynamically, instead of having the whole flow baked into +the system prompt. +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any, Optional + +from reachy_mini_receptionist.receptionist_state import ReceptionState +from reachy_mini_receptionist.session_manager import SessionManager + +logger = logging.getLogger(__name__) + + +# States in which an "unknown" face transitioning to "known" should still +# short-circuit identification (we haven't yet confirmed the visitor's name +# through other channels). +_EARLY_IDENTIFICATION_STATES: frozenset[ReceptionState] = frozenset({ + ReceptionState.IDLE, + ReceptionState.VISITOR_DETECTED, + ReceptionState.GREETING, + ReceptionState.ASK_NAME, + # Include MULTIPLE_PEOPLE so the controller transitions back out as + # soon as the crowd thins to one face. + ReceptionState.MULTIPLE_PEOPLE, +}) + +# States in which losing the face means the visitor walked away and we should +# reset the session for the next person. Inside flow states (e.g. ASK_NAME), +# losing the face briefly just means they turned their head — don't reset. +_RESET_ON_FACE_LOST_STATES: frozenset[ReceptionState] = frozenset({ + ReceptionState.NOTIFIED, + ReceptionState.NO_APPOINTMENT, + ReceptionState.EMAIL_FAILED, + ReceptionState.COMPLETE, + ReceptionState.ERROR, +}) + + +class ConversationController: + """Wire face events + tool completions into session state transitions. + + Stateless aside from the SessionManager it operates on. Safe to call + handlers from any thread because SessionManager handles locking. + + Optionally records every completed visit to a ``VisitorLog`` when the + session resets out of a meaningful state (visitor name, recognized + face, or employee was set). + """ + + def __init__(self, session_manager: SessionManager) -> None: + self._session = session_manager + + # ------------------------------------------------------------------ + # Face events + # ------------------------------------------------------------------ + + def on_face_event(self, event: dict[str, Any]) -> None: + """Translate a ``face_state_changed`` event into a session transition. + + Event shape (from ``FaceRecognitionWorker._update_stable_state``): + state: "no_face" | "unknown" | "known" + name: str | None (populated when state == "known") + previous_state, previous_name, lbph_confidence, detection_confidence + """ + state = event.get("state") + name = event.get("name") + current = self._session.current_state + snapshot = self._session.session + + if state == "known" and name: + # Already-completed-flow guard: if we already emailed the host + # for this visitor, don't re-enter RECOGNIZED — that re-triggers + # the whole check-in (get_today_calendar + send_email) and + # produces duplicate emails when the face momentarily flickers + # to MULTIPLE_PEOPLE and back. + already_done = ( + bool(snapshot.email_sent_to) + and (snapshot.visitor_name or "").strip().lower() == name.strip().lower() + ) + if already_done: + if current != ReceptionState.NOTIFIED: + logger.info( + "Face %r returned after notified — restoring NOTIFIED instead of re-running flow", + name, + ) + self._session.transition( + ReceptionState.NOTIFIED, + recognized_face_name=name, + ) + else: + self._session.update(recognized_face_name=name) + return + + if current in _EARLY_IDENTIFICATION_STATES: + # If the visitor has ALREADY told us a different name in + # this session (visitor_name set by register_guest), trust + # speech over face. The face recognizer can mis-match + # (LBPH on a single crop is noisy under different lighting) + # and the visitor explicitly correcting "no, I'm X" should + # override the camera. Only auto-promote face -> visitor + # when no speech-confirmed name exists yet. + speech_confirmed = (snapshot.visitor_name or "").strip() + if speech_confirmed and speech_confirmed.lower() != name.strip().lower(): + logger.info( + "Face matched %r but visitor already confirmed %r — keeping speech", + name, speech_confirmed, + ) + self._session.update(recognized_face_name=name) + return + self._session.transition( + ReceptionState.RECOGNIZED, + visitor_name=name, + recognized_face_name=name, + ) + # Same auto-resolve as the register_guest path so the + # face-recognition shortcut also reaches APPOINTMENT_MATCHED + # without depending on the LLM to call get_today_calendar. + try: + self._auto_resolve_appointment(name) + except Exception as e: + logger.warning( + "Auto-resolve appointment after face match failed: %s", e, + ) + else: + # Past identification — just record the face match. + self._session.update(recognized_face_name=name) + return + + if state == "unknown": + if current in (ReceptionState.IDLE, ReceptionState.MULTIPLE_PEOPLE): + self._session.transition(ReceptionState.VISITOR_DETECTED) + return + + if state == "multiple": + if current != ReceptionState.MULTIPLE_PEOPLE: + self._session.transition(ReceptionState.MULTIPLE_PEOPLE) + return + + if state == "no_face": + if current in _RESET_ON_FACE_LOST_STATES: + logger.info("Face lost in terminal state %s — resetting session", current.value) + # SessionManager.reset() handles persisting the pre-reset + # snapshot to the visitor log on its own. + self._session.reset() + return + + # ------------------------------------------------------------------ + # Tool completions + # ------------------------------------------------------------------ + + async def on_tool_completed_async( + self, + tool_name: str, + args: dict[str, Any], + result: dict[str, Any], + ) -> None: + """Async-safe wrapper for ``on_tool_completed``. + + The realtime event loop reaches the controller via this method. + Tools whose handlers need an iCal fetch (``register_guest`` triggers + ``_auto_resolve_appointment``) pre-fetch the calendar via + ``asyncio.to_thread`` so the audio loop never blocks on the + synchronous httpx call inside ``ical_calendar.fetch_appointments``. + """ + if not isinstance(result, dict): + return + explicit_failure = "error" in result and result.get("success") is False + if explicit_failure: + self.on_tool_completed(tool_name, args, result) + return + + appointments: Optional[list[dict[str, Any]]] = None + if tool_name in ("register_guest", "lookup_employee"): + try: + from reachy_mini_receptionist import calendar_data + appointments = await calendar_data.get_appointments_async() + except Exception as e: + logger.debug("Pre-fetch appointments failed: %s", e) + appointments = None + + self._dispatch_tool_completion(tool_name, args, result, appointments) + + def on_tool_completed( + self, + tool_name: str, + args: dict[str, Any], + result: dict[str, Any], + ) -> None: + """Translate a successful tool call into a session transition. + + Failures are logged but never transition the session into ERROR + automatically — that's the caller's policy choice. + + Synchronous variant — calls into ``_auto_resolve_appointment`` will + block on the iCal HTTP fetch. Safe from background threads (face + worker). Async callers on the realtime event loop should use + ``on_tool_completed_async`` instead so the iCal call gets + off-thread. + """ + if not isinstance(result, dict): + return + explicit_failure = "error" in result and result.get("success") is False + if explicit_failure: + logger.debug("Tool %s reported failure: %s", tool_name, result.get("error")) + return + self._dispatch_tool_completion(tool_name, args, result, None) + + def _dispatch_tool_completion( + self, + tool_name: str, + args: dict[str, Any], + result: dict[str, Any], + appointments: Optional[list[dict[str, Any]]], + ) -> None: + """Core transition logic shared by sync + async entry points. + + ``appointments`` is the optional pre-fetched calendar (set by the + async entry point so the iCal HTTP call doesn't run on the + realtime audio loop). When ``None``, ``_auto_resolve_appointment`` + falls back to its own sync fetch. + """ + if tool_name == "register_guest": + # Only transition on actual SUCCESS. If register_guest was + # blocked (no_confirmation, name_is_filler, hallucinated + # chatter, no_face, etc.) it returns success=False — we + # MUST NOT advance the session in that case, or the visitor + # ends up locked into a bogus name like "Community" with + # no path to fix it. + if not result.get("success"): + logger.debug( + "register_guest returned success=False (reason=%r) — not transitioning", + result.get("blocked_reason") or result.get("error"), + ) + return + name = (args.get("name") or "").strip() + if name: + self._session.transition( + ReceptionState.RECOGNIZED, + visitor_name=name, + recognized_face_name=name, + ) + # The LLM was supposed to follow the RECOGNIZED hint with a + # get_today_calendar tool call, but it kept asking the visitor + # "who are you here to see?" instead — emails never went out. + # Pull the calendar synchronously from the backend and dispatch + # APPOINTMENT_MATCHED / NO_APPOINTMENT ourselves so the bot is + # never blocked on the LLM remembering to look something up. + try: + self._auto_resolve_appointment(name, appointments) + except Exception as e: + logger.warning( + "Auto-resolve appointment after register_guest failed: %s", e, + ) + + elif tool_name == "get_today_calendar": + calendar = result.get("calendar") or [] + snap = self._session.session + # Prefer explicit visitor_name (operator typed/spoke it), fall + # back to a recognized face match (returning guest whose name + # we already trust because LBPH matched their saved crop). + visitor_name = snap.visitor_name or snap.recognized_face_name + if not visitor_name: + # LLM fetched the calendar as a generic lookup (often during + # idle exploration) before identifying the visitor. There is + # no name to match against yet, so don't change state. + logger.debug( + "get_today_calendar fired without a visitor_name — skipping transition", + ) + return + matched = self._match_appointment(calendar, visitor_name) + updates: dict[str, Any] = {} + # If we matched purely off the face name, promote it into + # visitor_name so downstream (send_email guard, dashboard, + # visitor log) treats it as a confirmed identity. + if not snap.visitor_name: + updates["visitor_name"] = visitor_name + if matched: + updates["matched_appointment"] = matched + updates["employee_name"] = matched.get("visiting") + self._session.transition(ReceptionState.APPOINTMENT_MATCHED, **updates) + else: + updates["error_message"] = f"No appointment found for {visitor_name!r}" + self._session.transition(ReceptionState.NO_APPOINTMENT, **updates) + + elif tool_name == "send_email": + # Only flip to NOTIFIED if the tool actually succeeded. The + # send_email tool can refuse (placeholder address, no visitor + # identity yet, duplicate-blocked, Resend HTTP error) — those + # all return success=False, and the dashboard must not lie + # "NOTIFIED" when no email left the system. + to = (args.get("to") or "").strip() + send_ok = bool(result.get("success")) + if to and send_ok: + self._session.transition( + ReceptionState.NOTIFIED, + email_sent_to=to, + ) + elif to and not send_ok: + logger.info( + "send_email returned success=False (blocked_reason=%s) — " + "NOT transitioning to NOTIFIED", + result.get("blocked_reason") or result.get("error"), + ) + + elif tool_name == "lookup_employee": + # Walk-in path: visitor named the host instead of themselves. + # On hit, drop the synthetic appointment into the session so the + # existing APPOINTMENT_MATCHED -> send_email -> NOTIFIED path + # works unchanged. On miss, surface UNKNOWN_EMPLOYEE so the + # bot tells the visitor that name isn't on the list. + found = bool(result.get("found")) + if found: + emp = result.get("employee") or {} + emp_email = (emp.get("email") or "").strip() + emp_name = (emp.get("name") or args.get("name") or "").strip() + if emp_email: + snap = self._session.session + # Trust the face DB: if LBPH already recognised this + # visitor (recognized_face_name), promote that into + # visitor_name so send_email's identity guard passes + # without forcing the bot to ask the name again. This + # is the "returning known guest came back to see X" + # path — they shouldn't be re-prompted for their name. + visitor = snap.visitor_name or snap.recognized_face_name + + # If the visitor is known AND today's calendar has a real + # appointment for them with this host, prefer that real + # appointment over a synthetic walk-in. Otherwise the + # host's notification email loses the scheduled time/note + # and reads "Walk-in visitor has arrived" for a meeting + # that was actually on the calendar. + real_appt: Optional[dict[str, Any]] = None + if visitor and appointments: + candidate = self._match_appointment(appointments, visitor) + if ( + candidate + and (candidate.get("visiting") or "").strip().lower() + == emp_email.lower() + ): + real_appt = candidate + + if real_appt is not None: + matched_appt = real_appt + else: + matched_appt = { + "time": "now", + "name": visitor or "Walk-in visitor", + "note": f"Walk-in to see {emp_name}", + "visiting": emp_email, + } + updates: dict[str, Any] = { + "matched_appointment": matched_appt, + "employee_name": emp_email, + } + if visitor and not snap.visitor_name: + updates["visitor_name"] = visitor + self._session.transition(ReceptionState.APPOINTMENT_MATCHED, **updates) + else: + query = (args.get("name") or "").strip() + self._session.transition( + ReceptionState.UNKNOWN_EMPLOYEE, + error_message=f"No directory match for {query!r}", + ) + + @staticmethod + def _match_appointment( + calendar: list[dict[str, Any]], + visitor_name: Optional[str], + ) -> Optional[dict[str, Any]]: + """Case-insensitive name match against today's calendar entries. + + Matching is layered so a visitor who says just their first name + ("Rohan") still resolves to a calendar entry like "Rohan Verma": + + 1. Exact match on the full string. + 2. Calendar entry's first whitespace-delimited token equals the + visitor string ("Rohan" == first(\"Rohan Verma\")). + 3. Substring of the calendar entry (\"rohan\" in \"rohan verma\"). + + Each layer returns the FIRST hit so we never silently switch which + calendar entry a visitor is mapped to. The minimum length guard + (>= 2 chars) keeps single-letter transcripts from matching half + the calendar. + """ + if not visitor_name: + return None + target = visitor_name.strip().lower() + if len(target) < 2: + return None + for appt in calendar: + if (appt.get("name") or "").strip().lower() == target: + return appt + for appt in calendar: + name = (appt.get("name") or "").strip().lower() + tokens = name.split() + if tokens and tokens[0] == target: + return appt + for appt in calendar: + name = (appt.get("name") or "").strip().lower() + if target in name: + return appt + return None + + def _auto_resolve_appointment( + self, + visitor_name: str, + appointments: Optional[list[dict[str, Any]]] = None, + ) -> None: + """Look up today's calendar for ``visitor_name`` and dispatch. + + Called from the RECOGNIZED transition in both the register_guest + path and the face-recognition path so the bot doesn't have to + depend on the LLM calling get_today_calendar — production showed + the LLM acknowledging the visitor and then improvising next-step + questions instead of running the tool, so the email never went out. + + ``appointments`` is an optional pre-fetched list. Async callers on + the realtime event loop preload it via ``calendar_data.get_appointments_async`` + so the synchronous iCal HTTP call doesn't block the audio loop here. + When omitted, falls back to a sync fetch (safe from background threads + like the face worker). + + Dispatches: + - APPOINTMENT_MATCHED with the matched appointment + host email, + if today's iCal has an entry whose ``name`` matches. + - NO_APPOINTMENT otherwise. The bot then offers to take a message + or route via lookup_employee. + """ + if not visitor_name: + return + if appointments is None: + from reachy_mini_receptionist import calendar_data + appointments = calendar_data.get_appointments() + matched = self._match_appointment(appointments, visitor_name) + if matched: + self._session.transition( + ReceptionState.APPOINTMENT_MATCHED, + matched_appointment=matched, + employee_name=matched.get("visiting"), + ) + logger.info( + "Auto-resolved appointment for %r -> %s", + visitor_name, matched.get("visiting"), + ) + else: + self._session.transition( + ReceptionState.NO_APPOINTMENT, + error_message=f"No appointment found for {visitor_name!r}", + ) + logger.info("Auto-resolve: no appointment for %r", visitor_name) + + +# ---------------------------------------------------------------------- +# Per-state workflow hints +# ---------------------------------------------------------------------- +# These get appended to the session context push so the LLM knows what to +# do next — without the workflow being hardcoded in the system prompt. +# Keep each hint short (one or two sentences) and concrete. The LLM has +# already been told to wait for the user to speak before responding; these +# hints describe what to do *when* the user speaks. + +_NEXT_ACTION_HINTS: dict[ReceptionState, str] = { + # IDLE is the "no visitor yet" state — normally we stay silent. BUT if a + # visitor speaks before the face worker has stabilised (camera obscured, + # off-angle, dim light), the bot would otherwise just greet generically + # and never advance. So when the user speaks during IDLE, treat the + # utterance as the start of the flow and dispatch immediately to the + # right tool based on what they said. + ReceptionState.IDLE: ( + "If the visitor speaks first, just be conversational — greet them " + "and figure out who they are or who they want to see. If they named " + "themselves, confirm once ('I heard , right?') then call " + "register_guest(name, confirmed=true). If they named a host, call " + "lookup_employee. ALWAYS respond — never go silent. If you mishear, " + "say so and ask again naturally; don't lecture." + ), + ReceptionState.VISITOR_DETECTED: ( + "Greet the visitor warmly. Ask their name or who they're here to see " + "if they haven't said yet. When they tell you a name, repeat it back " + "briefly to confirm; if they say yes, call register_guest with the " + "EXACT name you heard from the visitor and confirmed=true. Never " + "invent a name. If they're here to see someone, call lookup_employee. " + "Be conversational — short, friendly replies. ALWAYS respond to " + "whatever they say; never go silent." + ), + ReceptionState.GREETING: ( + "Greet the visitor. If they haven't said why they're here yet, " + "ask whether they have an appointment or are here to see someone." + ), + ReceptionState.ASK_NAME: ( + "Ask the visitor their name or who they're here to see. When they " + "answer, confirm the name back briefly and if they say yes, call " + "register_guest(confirmed=true). If you genuinely couldn't hear, " + "ask once more naturally. Keep replies short and friendly." + ), + ReceptionState.MULTIPLE_PEOPLE: ( + "More than one face is in view. Say 'I see more than one person — could " + "whoever's checking in step forward please?' Do NOT call register_guest, " + "lookup_employee, or send_email until the state changes back." + ), + ReceptionState.RECOGNIZED: ( + "Acknowledge the visitor by name. " + "Then call get_today_calendar to look up their appointment." + ), + ReceptionState.CHECKING_APPOINTMENT: ( + "Briefly let the visitor know you're checking the schedule." + ), + ReceptionState.APPOINTMENT_MATCHED: ( + "Use the appointment= and employee= values from the context above. " + "Say something like: 'Great, I have you down for with " + " — I'll let them know you're here.' Then immediately call " + "send_email to that host. If visitor= is empty, ask their name first." + ), + ReceptionState.NO_APPOINTMENT: ( + "Politely tell the visitor you don't have them on today's schedule. " + "Offer to take a message or notify someone." + ), + ReceptionState.NOTIFYING_EMPLOYEE: ( + "Briefly tell the visitor you're notifying their host." + ), + ReceptionState.NOTIFIED: ( + "Use the employee= from context. Say: 'Done — I've notified . " + "Please have a seat, they'll be with you shortly.' Be warm, not robotic." + ), + ReceptionState.EMAIL_FAILED: ( + "Apologize that you couldn't reach the host right now. " + "Suggest the visitor wait briefly while you try again." + ), + ReceptionState.WAITING: "", + ReceptionState.COMPLETE: ( + "Thank the visitor warmly and wish them a good day." + ), + ReceptionState.UNKNOWN_EMPLOYEE: ( + "Tell the visitor that name isn't in your directory. " + "Offer to find someone else who can help." + ), + ReceptionState.ERROR: ( + "Apologize for the issue and ask the visitor to wait a moment." + ), +} + + +def next_action_hint(state: ReceptionState) -> str: + """Return a short workflow directive for the LLM based on the current state.""" + return _NEXT_ACTION_HINTS.get(state, "") + + +# States that require the bot to speak IMMEDIATELY when entered, because the +# transition was triggered by an in-flight LLM response cycle (tool returned, +# state advanced — visitor is waiting for the bot to finish what it started). +# All others (face events, idle/reset) wait for the visitor to speak first. +_SPEAK_NOW_STATES: frozenset[ReceptionState] = frozenset({ + # Greet the visitor as soon as the face is detected. Previously the bot + # would silently wait for the visitor to speak first, which gave the + # impression of an unresponsive robot — visitors often hesitate when + # they don't know if the bot is "on" yet. + ReceptionState.VISITOR_DETECTED, + ReceptionState.RECOGNIZED, + ReceptionState.APPOINTMENT_MATCHED, + ReceptionState.NO_APPOINTMENT, + ReceptionState.NOTIFIED, + ReceptionState.EMAIL_FAILED, + ReceptionState.UNKNOWN_EMPLOYEE, +}) + + +def should_speak_immediately(state: ReceptionState) -> bool: + """True if entering ``state`` should trigger an immediate spoken response. + + For these states the LLM is mid-flow (just ran a tool, state advanced) + and the visitor is waiting. For all other states (face events, + timeouts, manual resets) the bot waits for the visitor to speak. + """ + return state in _SPEAK_NOW_STATES diff --git a/src/reachy_mini_receptionist/dance_emotion_moves.py b/src/reachy_mini_receptionist/dance_emotion_moves.py new file mode 100644 index 0000000000000000000000000000000000000000..a39dd67c7f694debfc9954023c845309331b3865 --- /dev/null +++ b/src/reachy_mini_receptionist/dance_emotion_moves.py @@ -0,0 +1,154 @@ +"""Dance and emotion moves for the movement queue system. + +This module implements dance moves and emotions as Move objects that can be queued +and executed sequentially by the MovementManager. +""" + +from __future__ import annotations +import logging +from typing import Tuple + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini.motion.move import Move +from reachy_mini.motion.recorded_move import RecordedMoves +from reachy_mini_dances_library.dance_move import DanceMove + + +logger = logging.getLogger(__name__) + + +class DanceQueueMove(Move): # type: ignore + """Wrapper for dance moves to work with the movement queue system.""" + + def __init__(self, move_name: str): + """Initialize a DanceQueueMove.""" + self.dance_move = DanceMove(move_name) + self.move_name = move_name + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return float(self.dance_move.duration) + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate dance move at time t.""" + try: + # Get the pose from the dance move + head_pose, antennas, body_yaw = self.dance_move.evaluate(t) + + # Convert to numpy array if antennas is tuple and return in official Move format + if isinstance(antennas, tuple): + antennas = np.array([antennas[0], antennas[1]]) + + return (head_pose, antennas, body_yaw) + + except Exception as e: + logger.error(f"Error evaluating dance move '{self.move_name}' at t={t}: {e}") + # Return neutral pose on error + from reachy_mini.utils import create_head_pose + + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + return (neutral_head_pose, np.array([0.0, 0.0], dtype=np.float64), 0.0) + + +class EmotionQueueMove(Move): # type: ignore + """Wrapper for emotion moves to work with the movement queue system.""" + + def __init__(self, emotion_name: str, recorded_moves: RecordedMoves): + """Initialize an EmotionQueueMove.""" + self.emotion_move = recorded_moves.get(emotion_name) + self.emotion_name = emotion_name + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return float(self.emotion_move.duration) + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate emotion move at time t.""" + try: + # Get the pose from the emotion move + head_pose, antennas, body_yaw = self.emotion_move.evaluate(t) + + # Convert to numpy array if antennas is tuple and return in official Move format + if isinstance(antennas, tuple): + antennas = np.array([antennas[0], antennas[1]]) + + return (head_pose, antennas, body_yaw) + + except Exception as e: + logger.error(f"Error evaluating emotion '{self.emotion_name}' at t={t}: {e}") + # Return neutral pose on error + from reachy_mini.utils import create_head_pose + + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + return (neutral_head_pose, np.array([0.0, 0.0], dtype=np.float64), 0.0) + + +class GotoQueueMove(Move): # type: ignore + """Wrapper for goto moves to work with the movement queue system.""" + + def __init__( + self, + target_head_pose: NDArray[np.float32], + start_head_pose: NDArray[np.float32] | None = None, + target_antennas: Tuple[float, float] = (0, 0), + start_antennas: Tuple[float, float] | None = None, + target_body_yaw: float = 0, + start_body_yaw: float | None = None, + duration: float = 1.0, + ): + """Initialize a GotoQueueMove.""" + self._duration = duration + self.target_head_pose = target_head_pose + self.start_head_pose = start_head_pose + self.target_antennas = target_antennas + self.start_antennas = start_antennas or (0, 0) + self.target_body_yaw = target_body_yaw + self.start_body_yaw = start_body_yaw or 0 + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return self._duration + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate goto move at time t using linear interpolation.""" + try: + from reachy_mini.utils import create_head_pose + from reachy_mini.utils.interpolation import linear_pose_interpolation + + # Clamp t to [0, 1] for interpolation + t_clamped = max(0, min(1, t / self.duration)) + + # Use start pose if available, otherwise neutral + if self.start_head_pose is not None: + start_pose = self.start_head_pose + else: + start_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + + # Interpolate head pose + head_pose = linear_pose_interpolation(start_pose, self.target_head_pose, t_clamped) + + # Interpolate antennas - return as numpy array + antennas = np.array( + [ + self.start_antennas[0] + (self.target_antennas[0] - self.start_antennas[0]) * t_clamped, + self.start_antennas[1] + (self.target_antennas[1] - self.start_antennas[1]) * t_clamped, + ], + dtype=np.float64, + ) + + # Interpolate body yaw + body_yaw = self.start_body_yaw + (self.target_body_yaw - self.start_body_yaw) * t_clamped + + return (head_pose, antennas, body_yaw) + + except Exception as e: + logger.error(f"Error evaluating goto move at t={t}: {e}") + # Return target pose on error - convert to float64 + target_head_pose_f64 = self.target_head_pose.astype(np.float64) + target_antennas_array = np.array([self.target_antennas[0], self.target_antennas[1]], dtype=np.float64) + return (target_head_pose_f64, target_antennas_array, self.target_body_yaw) diff --git a/src/reachy_mini_receptionist/employees.py b/src/reachy_mini_receptionist/employees.py new file mode 100644 index 0000000000000000000000000000000000000000..16ed093a8acba9259f3c1969cd61d829e95840b8 --- /dev/null +++ b/src/reachy_mini_receptionist/employees.py @@ -0,0 +1,121 @@ +"""Employee directory — read API used by tools, calendar, and the LLM. + +Two-tier source: + +1. **Primary**: ``EmployeeStore`` (SQLite), populated and edited via the + dashboard's Employees panel. +2. **Fallback**: the hardcoded ``_SEED_EMPLOYEES`` constant below. This + only takes effect when the store is unset (e.g. tests that import + ``employees`` directly) OR when the store is empty. On a real + deployment, ``main.py`` constructs the store and seeds it with + ``_SEED_EMPLOYEES`` on first run; after that, edits via the + dashboard are the source of truth. + +Consumers (``lookup_employee``, ``find_email_for``, ``get_all_employees``, +``format_for_llm``) keep the same signatures so ``calendar_data`` and the +``lookup_employee`` tool need zero changes. +""" +from __future__ import annotations + +import logging +from typing import Any, List, Optional, TypedDict + +logger = logging.getLogger(__name__) + + +class Employee(TypedDict, total=False): + name: str + email: str + aliases: List[str] + title: Optional[str] + + +# Seed list intentionally empty. A brand-new install starts with zero +# employees and the dashboard's Employees panel shows an empty state +# with a "+ Add" button — that's a clearer first-run UX than pre-loading +# dummy entries the operator has to delete one by one before adding +# their real team. The seed-load mechanism in employees_store.py is +# preserved for callers that want to bundle a seed list (e.g. a future +# import-from-CSV path). +_SEED_EMPLOYEES: List[Employee] = [] + + +# Process-wide reference to the active store. ``main.py`` calls +# ``set_store(...)`` after construction. Kept ``None`` in tests / imports +# that don't go through main, in which case we fall back to the seed list. +_store: Any = None + + +def set_store(store: Any) -> None: + """Register the EmployeeStore the module should read from.""" + global _store + _store = store + logger.info("employees: bound to store (count=%s)", store.count() if store else "n/a") + + +def _strip_internal(emp: dict[str, Any]) -> Employee: + """Drop SQLite-side fields the LLM doesn't need (id, created_at, etc).""" + return { # type: ignore[return-value] + "name": emp.get("name", ""), + "email": emp.get("email", ""), + "aliases": list(emp.get("aliases") or []), + "title": emp.get("title"), + } + + +def _normalize(s: str) -> str: + return (s or "").strip().lower() + + +def get_all_employees() -> List[Employee]: + """Return a snapshot of the full directory.""" + if _store is not None: + try: + rows = _store.list_all() + if rows: + return [_strip_internal(r) for r in rows] + except Exception as e: + logger.warning("employees.get_all_employees: store read failed (%s); using seed", e) + return [dict(e) for e in _SEED_EMPLOYEES] # type: ignore[misc] + + +def lookup_employee(query: str) -> Optional[Employee]: + """Find an employee by name or alias (case-insensitive, exact-only).""" + if not (query or "").strip(): + return None + if _store is not None: + try: + hit = _store.lookup(query) + if hit: + return _strip_internal(hit) + except Exception as e: + logger.warning("employees.lookup_employee: store read failed (%s); using seed", e) + q = _normalize(query) + if not q: + return None + for emp in _SEED_EMPLOYEES: + if _normalize(emp.get("name", "")) == q: + return dict(emp) # type: ignore[return-value] + for alias in emp.get("aliases", []) or []: + if _normalize(alias) == q: + return dict(emp) # type: ignore[return-value] + return None + + +def find_email_for(query: str) -> Optional[str]: + """Convenience: resolve a name/alias to an email, or None.""" + emp = lookup_employee(query) + return emp.get("email") if emp else None + + +def format_for_llm() -> str: + """Render the directory as a short string the LLM can reference.""" + employees = get_all_employees() + if not employees: + return "Employee directory is empty." + lines = ["Employee directory:"] + for emp in employees: + aliases = emp.get("aliases") or [] + alias_str = f" (also: {', '.join(aliases)})" if aliases else "" + lines.append(f" - {emp.get('name', '?')}{alias_str}") + return "\n".join(lines) diff --git a/src/reachy_mini_receptionist/employees_store.py b/src/reachy_mini_receptionist/employees_store.py new file mode 100644 index 0000000000000000000000000000000000000000..0280a88f0839d1e0274e195ee87c24ab9afc57f1 --- /dev/null +++ b/src/reachy_mini_receptionist/employees_store.py @@ -0,0 +1,342 @@ +"""SQLite-backed employee directory CRUD. + +The dashboard's Employees panel uses this; ``employees.py`` reads from +this store too (with a fall-through to the hardcoded ``_EMPLOYEES`` list +when the store is empty, which only happens on a brand-new install +before the seed runs). + +Schema matches the operator's mental model: name, email, optional title, +optional list of aliases the bot might hear (e.g. "AJ" for "Arjun +Mehta"). Name uniqueness is enforced case-insensitively so the bot can +never end up with two "Mukul" entries that route differently. + +Lives next to ``visitor_log.db`` in the app's instance directory. Uses +the same WAL + per-call connection pattern. +""" +from __future__ import annotations + +import json +import logging +import sqlite3 +import threading +from datetime import datetime +from pathlib import Path +from typing import Any, Iterable, List, Optional + +logger = logging.getLogger(__name__) + + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS employees ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + email TEXT NOT NULL, + title TEXT, + aliases TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_employees_name_lower + ON employees (LOWER(name)); +""" + + +class EmployeeExistsError(Exception): + """Raised when an employee name (case-insensitive) already exists.""" + + +class EmployeeNotFoundError(Exception): + """Raised when a CRUD operation targets a missing employee id.""" + + +class EmployeeStore: + """Thread-safe employee directory backed by a single SQLite file.""" + + def __init__(self, db_path: str | Path) -> None: + self._db_path = Path(db_path) + self._db_path.parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self._init_schema() + logger.info("EmployeeStore initialised at %s", self._db_path) + + # ------------------------------------------------------------------ + # Connection / schema + # ------------------------------------------------------------------ + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self._db_path, check_same_thread=False, timeout=5.0) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA synchronous=NORMAL") + return conn + + def _init_schema(self) -> None: + with self._lock: + conn = self._connect() + try: + conn.execute("PRAGMA journal_mode=WAL") + conn.executescript(_SCHEMA) + conn.commit() + finally: + conn.close() + + # ------------------------------------------------------------------ + # Row <-> dict helpers + # ------------------------------------------------------------------ + + @staticmethod + def _row_to_dict(row: sqlite3.Row) -> dict[str, Any]: + aliases_raw = row["aliases"] or "[]" + try: + aliases = json.loads(aliases_raw) + if not isinstance(aliases, list): + aliases = [] + except Exception: + aliases = [] + return { + "id": row["id"], + "name": row["name"], + "email": row["email"], + "title": row["title"], + "aliases": aliases, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + @staticmethod + def _aliases_to_text(aliases: Optional[Iterable[str]]) -> str: + cleaned = [] + seen: set[str] = set() + for a in aliases or []: + s = (a or "").strip() + if not s: + continue + key = s.lower() + if key in seen: + continue + seen.add(key) + cleaned.append(s) + return json.dumps(cleaned) + + # ------------------------------------------------------------------ + # Reads + # ------------------------------------------------------------------ + + def list_all(self) -> List[dict[str, Any]]: + with self._lock: + conn = self._connect() + try: + rows = conn.execute( + "SELECT * FROM employees ORDER BY LOWER(name)", + ).fetchall() + finally: + conn.close() + return [self._row_to_dict(r) for r in rows] + + def count(self) -> int: + with self._lock: + conn = self._connect() + try: + row = conn.execute("SELECT COUNT(*) AS n FROM employees").fetchone() + finally: + conn.close() + return int(row["n"]) if row else 0 + + def get_by_id(self, employee_id: int) -> Optional[dict[str, Any]]: + with self._lock: + conn = self._connect() + try: + row = conn.execute( + "SELECT * FROM employees WHERE id = ?", + (int(employee_id),), + ).fetchone() + finally: + conn.close() + return self._row_to_dict(row) if row else None + + def lookup(self, query: str) -> Optional[dict[str, Any]]: + """Find an employee by name OR alias (case-insensitive, exact match). + + Mirrors the original ``employees.lookup_employee`` semantics — exact + match only so that "Sam" can never silently route to "Samira". + """ + q = (query or "").strip().lower() + if not q: + return None + with self._lock: + conn = self._connect() + try: + rows = conn.execute( + "SELECT * FROM employees WHERE LOWER(name) = ?", + (q,), + ).fetchall() + if rows: + return self._row_to_dict(rows[0]) + all_rows = conn.execute("SELECT * FROM employees").fetchall() + finally: + conn.close() + for row in all_rows: + d = self._row_to_dict(row) + for alias in d.get("aliases") or []: + if (alias or "").strip().lower() == q: + return d + return None + + # ------------------------------------------------------------------ + # Writes + # ------------------------------------------------------------------ + + def create( + self, + name: str, + email: str, + aliases: Optional[Iterable[str]] = None, + title: Optional[str] = None, + ) -> dict[str, Any]: + name = (name or "").strip() + email = (email or "").strip() + if not name: + raise ValueError("name is required") + if not email: + raise ValueError("email is required") + aliases_text = self._aliases_to_text(aliases) + now = datetime.utcnow().isoformat(timespec="seconds") + with self._lock: + conn = self._connect() + try: + try: + cur = conn.execute( + """ + INSERT INTO employees (name, email, title, aliases, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + """, + (name, email, (title or None), aliases_text, now, now), + ) + conn.commit() + new_id = cur.lastrowid + except sqlite3.IntegrityError as e: + raise EmployeeExistsError( + f"An employee named {name!r} already exists (case-insensitive)" + ) from e + row = conn.execute( + "SELECT * FROM employees WHERE id = ?", + (new_id,), + ).fetchone() + finally: + conn.close() + logger.info("EmployeeStore.create: id=%s name=%r email=%r", new_id, name, email) + return self._row_to_dict(row) + + def update( + self, + employee_id: int, + *, + name: Optional[str] = None, + email: Optional[str] = None, + aliases: Optional[Iterable[str]] = None, + title: Optional[str] = None, + ) -> Optional[dict[str, Any]]: + sets: List[str] = [] + params: List[Any] = [] + if name is not None: + cleaned = name.strip() + if not cleaned: + raise ValueError("name cannot be empty") + sets.append("name = ?") + params.append(cleaned) + if email is not None: + cleaned = email.strip() + if not cleaned: + raise ValueError("email cannot be empty") + sets.append("email = ?") + params.append(cleaned) + if aliases is not None: + sets.append("aliases = ?") + params.append(self._aliases_to_text(aliases)) + if title is not None: + sets.append("title = ?") + params.append(title.strip() or None) + if not sets: + # Nothing to update; return current row. + return self.get_by_id(employee_id) + sets.append("updated_at = ?") + params.append(datetime.utcnow().isoformat(timespec="seconds")) + params.append(int(employee_id)) + with self._lock: + conn = self._connect() + try: + try: + cur = conn.execute( + f"UPDATE employees SET {', '.join(sets)} WHERE id = ?", + params, + ) + conn.commit() + if cur.rowcount == 0: + return None + except sqlite3.IntegrityError as e: + raise EmployeeExistsError( + "Another employee already uses that name (case-insensitive)" + ) from e + row = conn.execute( + "SELECT * FROM employees WHERE id = ?", + (int(employee_id),), + ).fetchone() + finally: + conn.close() + if row: + logger.info("EmployeeStore.update: id=%s -> %s", employee_id, dict(row)) + return self._row_to_dict(row) + return None + + def delete(self, employee_id: int) -> bool: + with self._lock: + conn = self._connect() + try: + cur = conn.execute( + "DELETE FROM employees WHERE id = ?", + (int(employee_id),), + ) + conn.commit() + removed = cur.rowcount > 0 + finally: + conn.close() + if removed: + logger.info("EmployeeStore.delete: id=%s", employee_id) + return removed + + # ------------------------------------------------------------------ + # Seeding + # ------------------------------------------------------------------ + + def seed_if_empty(self, employees: Iterable[dict[str, Any]]) -> int: + """Bulk-insert ``employees`` only if the table is currently empty. + + Returns the number inserted. Idempotent across restarts — the + seed runs once on a brand-new install and is then a no-op. + Duplicate-name conflicts inside the seed list are skipped (the + first occurrence wins) so partial seeds don't abort the whole + batch. + """ + if self.count() > 0: + return 0 + inserted = 0 + for emp in employees: + try: + self.create( + name=emp.get("name", ""), + email=emp.get("email", ""), + aliases=emp.get("aliases") or [], + title=emp.get("title"), + ) + inserted += 1 + except EmployeeExistsError: + logger.warning( + "Seed: skipping duplicate %r", emp.get("name"), + ) + except Exception as e: + logger.warning( + "Seed: failed to insert %r: %s", emp.get("name"), e, + ) + if inserted: + logger.info("EmployeeStore: seeded %d employee(s)", inserted) + return inserted diff --git a/src/reachy_mini_receptionist/face_db.py b/src/reachy_mini_receptionist/face_db.py new file mode 100644 index 0000000000000000000000000000000000000000..a8c3ecd3086b8033e10fce274e20a1b0050e5f8f --- /dev/null +++ b/src/reachy_mini_receptionist/face_db.py @@ -0,0 +1,184 @@ +"""File-based face store for OpenCV LBPH recognition. + +Design decisions: +- Stores 100×100 grayscale face crops as PNG files named after the guest. + e.g. guests/Beyonce.png, guests/Elon Musk.png +- No database required — files are the database. Easy to inspect, edit, + or delete with any file manager. +- Max 100 guests. When full, the oldest file (by mtime) is replaced (FIFO). +- Thread-safe: all writes use a threading.Lock. +- The guests/ directory lives in the app instance directory so it persists + across restarts. +""" +from __future__ import annotations + +import logging +import threading +from datetime import datetime +from pathlib import Path +from typing import List, Optional, Tuple + +import cv2 +import numpy as np + +logger = logging.getLogger(__name__) + +MAX_GUESTS = 100 + + +def _safe_filename(name: str) -> str: + """Sanitise a guest name so it is safe to use as a filename.""" + # Replace characters that are problematic on Windows/Linux/macOS + for ch in r'\/:*?"<>|': + name = name.replace(ch, "_") + return name.strip() or "unknown" + + +class FaceDatabase: + """File-based face crop store (PNG per guest). + + Public API is intentionally identical to the previous SQLite version so + that all callers (FaceRecognitionWorker, register_guest tool, API + endpoints) need zero changes. + """ + + def __init__(self, db_path: str | Path) -> None: + # Accept the old ``guests.db`` path and derive the sibling directory + # from it so the call-site in main.py doesn't need to change. + db_path = Path(db_path) + self._guests_dir = db_path.parent / "guests" + self._guests_dir.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + logger.info("FaceDatabase (file-based) initialised at %s", self._guests_dir) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _path_for(self, name: str) -> Path: + return self._guests_dir / f"{_safe_filename(name)}.png" + + def _all_png_files(self) -> List[Path]: + """Return all .png files sorted newest-first (by mtime).""" + files = list(self._guests_dir.glob("*.png")) + files.sort(key=lambda p: p.stat().st_mtime, reverse=True) + return files + + # ------------------------------------------------------------------ + # Write + # ------------------------------------------------------------------ + + def add_or_update_guest(self, name: str, face_crop: np.ndarray) -> None: + """Save a 100×100 grayscale face crop as ``/.png``. + + If the directory already has MAX_GUESTS different entries and ``name`` + is new, the oldest file (by mtime) is deleted first (FIFO eviction). + """ + target = self._path_for(name) + + with self._lock: + if not target.exists(): + files = list(self._guests_dir.glob("*.png")) + if len(files) >= MAX_GUESTS: + # Evict the oldest file + oldest = min(files, key=lambda p: p.stat().st_mtime) + oldest.unlink() + logger.info("Evicted oldest guest file: %s (capacity=%d)", oldest.name, MAX_GUESTS) + + ok = cv2.imwrite(str(target), face_crop) + if ok: + logger.info("Saved guest '%s' → %s", name, target) + else: + raise RuntimeError(f"cv2.imwrite failed for path: {target}") + + # ------------------------------------------------------------------ + # Read + # ------------------------------------------------------------------ + + def get_all_guests(self) -> List[dict]: + """Return all guests as dicts with keys: name, timestamp, thumbnail_url.""" + with self._lock: + files = self._all_png_files() + result = [] + for f in files: + mtime = f.stat().st_mtime + ts = datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") + guest_name = f.stem # filename without .png extension + result.append({ + "name": guest_name, + "timestamp": ts, + "thumbnail_url": f"/guest_images/{f.name}", + }) + return result + + def get_all_guests_with_crops(self) -> List[Tuple[str, np.ndarray]]: + """Return list of (name, face_crop) for LBPH recognizer training.""" + with self._lock: + files = self._all_png_files() + result = [] + for f in files: + crop = cv2.imread(str(f), cv2.IMREAD_GRAYSCALE) + if crop is not None: + result.append((f.stem, crop)) + else: + logger.warning("Could not read face crop from %s — skipping", f) + return result + + def count(self) -> int: + with self._lock: + return len(list(self._guests_dir.glob("*.png"))) + + def clear(self) -> None: + """Wipe all guest images (useful for demo reset).""" + with self._lock: + for f in self._guests_dir.glob("*.png"): + f.unlink() + logger.info("FaceDatabase cleared") + + def cleanup_older_than(self, max_age_days: float) -> int: + """Delete guest PNGs whose mtime is older than ``max_age_days``. + + Returns the number of files removed. Pass 0 or negative to disable + cleanup (returns immediately). Failures to remove individual files + are logged but do not raise — TTL is best-effort. + """ + import time as _t + + if max_age_days <= 0: + return 0 + cutoff = _t.time() - (max_age_days * 86400.0) + removed = 0 + with self._lock: + for f in list(self._guests_dir.glob("*.png")): + try: + if f.stat().st_mtime < cutoff: + f.unlink() + removed += 1 + logger.info( + "Face TTL: removed %s (older than %.1f days)", + f.name, max_age_days, + ) + except Exception as e: + logger.warning("Face TTL: could not remove %s: %s", f, e) + return removed + + def delete_guest(self, name: str) -> bool: + """Delete one guest PNG by name. + + Returns True if the file existed and was removed, False otherwise. + """ + target = self._path_for(name) + with self._lock: + if not target.exists(): + return False + target.unlink() + logger.info("Deleted guest '%s' -> %s", name, target) + return True + + # ------------------------------------------------------------------ + # Expose the guests directory path (needed by main.py to mount statics) + # ------------------------------------------------------------------ + + @property + def guests_dir(self) -> Path: + return self._guests_dir diff --git a/src/reachy_mini_receptionist/face_recognition_worker.py b/src/reachy_mini_receptionist/face_recognition_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..440effe6b4536c11c5d9fed7f861210e8337b53f --- /dev/null +++ b/src/reachy_mini_receptionist/face_recognition_worker.py @@ -0,0 +1,698 @@ +"""Background face recognition worker — OpenCV YuNet detection + LBPH recognition. + +Design decisions: +- Uses cv2.FaceDetectorYN (YuNet) for face detection: returns a real confidence + score (0–1) per bounding box. Model (~400 KB ONNX) is downloaded to + ~/.cache/reachy_mini/ on first use. +- Uses OpenCV LBPH (Local Binary Pattern Histogram) recognizer for identification. + Same algorithm used in embedded/microcontroller face recognition. +- Runs in a daemon thread so it never blocks the audio/LLM loop. +- Processes every Nth frame to keep CPU usage low. +- Annotates frames with bounding boxes + labels for the MJPEG dashboard stream. +- All shared state is protected by threading.Lock so tools can read safely. +- Frames are pulled from CameraWorker (robot camera via SDK) rather than opening + a raw cv2.VideoCapture, so the correct camera is always used on Lite setups. + +Detection quality metrics: +- Detection confidence (YuNet score 0–1): higher = more certain this is a face. + Used as a multiplier in the quality score so uncertain detections rank lower. +- Blur score (Laplacian variance): cv2.Laplacian(crop, cv2.CV_64F).var() + High value = sharp / lots of edge detail → good crop. + Low value = blurry / uniform → bad crop (head mid-motion, etc.). + Crops below _MIN_BLUR_SCORE are considered low-quality, but they are still + stored so dashboard previews and fallback checks can return "best available" + evidence instead of empty results. +- Face area (w×h pixels) is kept as a secondary tiebreaker: among comparably sharp + crops the closer/larger face is still preferred. +- Combined quality score: blur_score × log(face_area) × max(det_confidence, 0.01) + +Requirements: opencv-contrib-python (pip install opencv-contrib-python) + The contrib package is a superset of opencv-python — don't install both. +""" +from __future__ import annotations + +import logging +import math +import pathlib +import threading +import time +import urllib.request +from collections import deque +from typing import Any, Callable, Optional + +import cv2 +import numpy as np + +logger = logging.getLogger(__name__) + +# How many frames to skip between recognition passes +_PROCESS_EVERY_N_FRAMES = 10 +# LBPH confidence threshold: LOWER = stricter match (distance, not similarity). +# +# Calibration notes (single 100×100 grayscale crop per guest): +# ≤ 50 almost certainly the same person (well-lit, same angle) +# 50 - 75 plausible match (different angle/lighting OK) +# 75 - 100 weak — often false positives on similar-looking people +# 100 - 160 printed photos, strangers who happen to resemble a guest +# > 160 unrelated face / no match +# +# Default was 110 which silently mis-recognised strangers as registered +# guests (we saw a stranger scored 103 and got greeted as "Jon"). 75 is +# the right starting point: prefers "I don't recognise you, please tell +# me your name" over a wrong-greeting failure. Operators can tune via +# the FACE_LBPH_THRESHOLD env var if their lighting/angles demand it. +_DEFAULT_CONFIDENCE_THRESHOLD = 75.0 + + +def _resolve_threshold() -> float: + """Read FACE_LBPH_THRESHOLD env var, falling back to the default.""" + import os + try: + raw = os.getenv("FACE_LBPH_THRESHOLD") + if raw is None or not str(raw).strip(): + return _DEFAULT_CONFIDENCE_THRESHOLD + val = float(raw) + if val <= 0: + return _DEFAULT_CONFIDENCE_THRESHOLD + return val + except Exception: + return _DEFAULT_CONFIDENCE_THRESHOLD + + +_CONFIDENCE_THRESHOLD = _resolve_threshold() +# Minimum Laplacian variance for a crop to be accepted into the detection window. +# Laplacian variance measures image sharpness: high = sharp, low = blurry. +# Typical values: sharp well-lit face 150–400, soft/distant face 60–120, +# head mid-motion 5–40, total blur < 5. +# Crops below this threshold are considered low quality and ranked lower. +_MIN_BLUR_SCORE = 80.0 # lower to ~50 if valid faces get rejected; raise to ~120 if blur sneaks through + +# Central detection zone: fractions of frame dimensions. +# Faces whose centre falls outside this zone are ignored. +# 0.25 margin → active zone = middle 50% horizontally and 80% vertically. +_ZONE_X_MARGIN = 0.25 # 25% margin on each side → 50% wide centre zone +_ZONE_Y_MARGIN = 0.10 # 10% margin on top/bottom → 80% tall centre zone + +# Rolling window duration (seconds) for selecting the best recent face crop. +_BEST_FACE_WINDOW_SECONDS = 5.0 +# Minimum spread (seconds) between first and last detection in the window. +# A face must have been continuously present for this long to be returned by +# best_recent_face(). Prevents a briefly passing face from overriding the +# person who has been standing in front of the robot for several seconds. +_MIN_DWELL_SECONDS = 1.5 + +# Stable identity transition settings for external face context events. +# Observed identity must remain unchanged for _FACE_STATE_CONFIRM_SECONDS before +# becoming the stable state, except no-face which uses a longer grace period. +_FACE_STATE_CONFIRM_SECONDS = 1.2 +_NO_FACE_CONFIRM_SECONDS = 2.5 +# Slightly longer dwell for multi-person before promoting — people walking +# past should not trip MULTIPLE_PEOPLE. +_MULTIPLE_PEOPLE_CONFIRM_SECONDS = 1.5 +# Minimum interval between emitted external face events. +_FACE_EVENT_COOLDOWN_SECONDS = 5.0 +# Minimum number of in-zone faces to be considered "multiple". +# Set absurdly high to effectively DISABLE MULTIPLE_PEOPLE state — in the +# pilot lobby, background people / posters were tripping the state too +# easily and the bot would go silent mid-conversation. The state has no +# SPEAK_NOW cue, so once triggered the visitor's speech is ignored until +# the camera clears. For a single-receptionist deployment the largest- +# face heuristic the worker already uses is enough to pick the visitor. +_MULTIPLE_PEOPLE_THRESHOLD = 999 + +# YuNet (cv2.FaceDetectorYN) detection settings +_YUNET_SCORE_THRESHOLD = 0.6 # minimum per-face detection confidence (0–1) +_YUNET_NMS_THRESHOLD = 0.3 # non-max suppression overlap threshold +_YUNET_TOP_K = 5000 # max candidate detections before NMS +_YUNET_MODEL_URL = ( + "https://github.com/opencv/opencv_zoo/raw/main/models/face_detection_yunet/" + "face_detection_yunet_2023mar.onnx" +) +_YUNET_CACHE_PATH = pathlib.Path.home() / ".cache" / "reachy_mini" / "face_detection_yunet_2023mar.onnx" + + +def _ensure_yunet_model() -> pathlib.Path: + """Download the YuNet ONNX model (~400 KB) to cache if not already present.""" + if not _YUNET_CACHE_PATH.exists(): + _YUNET_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) + logger.info("Downloading YuNet model → %s", _YUNET_CACHE_PATH) + urllib.request.urlretrieve(_YUNET_MODEL_URL, _YUNET_CACHE_PATH) + logger.info("YuNet model downloaded.") + return _YUNET_CACHE_PATH + + +# Check LBPH recognizer availability (needs opencv-contrib-python) +_LBPH_AVAILABLE = hasattr(cv2, "face") and hasattr(cv2.face, "LBPHFaceRecognizer_create") +if not _LBPH_AVAILABLE: + logger.warning( + "cv2.face.LBPHFaceRecognizer_create not found. " + "Install opencv-contrib-python: pip install opencv-contrib-python\n" + "Face recognition (identification) will be disabled; detection still works." + ) + + +def _build_lbph_recognizer(label_crops: list[tuple[int, np.ndarray]]) -> "cv2.face.LBPHFaceRecognizer | None": + """Train and return an LBPH recognizer from a list of (label_int, gray_crop) pairs.""" + if not _LBPH_AVAILABLE or not label_crops: + return None + recognizer = cv2.face.LBPHFaceRecognizer_create() + labels = np.array([lc[0] for lc in label_crops], dtype=np.int32) + crops = [lc[1] for lc in label_crops] + recognizer.train(crops, labels) + return recognizer + + +class FaceRecognitionWorker: + """Background thread that continuously detects and recognises faces. + + Public API (thread-safe): + worker.current_name → str "Unknown" or registered name + worker.current_encoding → Optional[np.ndarray] raw face crop (grayscale) + worker.latest_annotated_jpeg → Optional[bytes] MJPEG frame + worker.confidence → float 0–100 LBPH confidence (lower = better) + worker.best_recent_face() → (name, lbph_conf, crop) from 5-second window + """ + + def __init__( + self, + face_db, # FaceDatabase instance + camera_worker=None, # CameraWorker instance (pulls robot frames via SDK) + process_every_n: int = _PROCESS_EVERY_N_FRAMES, + confidence_threshold: float = _CONFIDENCE_THRESHOLD, + ) -> None: + self._face_db = face_db + self._camera_worker = camera_worker + self._process_every_n = process_every_n + self._confidence_threshold = confidence_threshold + + # Shared state (read by tools and dashboard) + self._lock = threading.Lock() + self._current_name: str = "Unknown" + self._current_encoding: Optional[np.ndarray] = None # grayscale face crop + self._latest_annotated_jpeg: Optional[bytes] = None + self._confidence: float = 0.0 + + # Stable face state for context event emission. + # state in {"no_face", "unknown", "known"} + now = time.monotonic() + self._stable_state: str = "no_face" + self._stable_name: str = "Unknown" + self._stable_since: float = now + self._candidate_state: str = "no_face" + self._candidate_name: str = "Unknown" + self._candidate_since: float = now + self._last_event_sent_at: float = 0.0 + self._face_event_callback: Optional[Callable[[dict[str, Any]], None]] = None + + # Rolling 5-second detection window: each entry is + # (timestamp: float, face_area: int, blur_score: float, crop: np.ndarray, det_confidence: float) + # face_area = w * h in pixels — larger = more prominent / closer face + # blur_score = Laplacian variance — higher = sharper crop + # det_confidence = YuNet score 0–1 — higher = more confident detection + # Entries with blur_score < _MIN_BLUR_SCORE are still added, but rank lower. + self._detection_window: deque = deque() + + # Log buffer for dashboard debug panel (ring buffer, max 200 lines) + self._log_buffer: list[str] = [] + self._log_lock = threading.Lock() + + # LBPH recognizer — rebuilt whenever guests are added/changed + self._recognizer_lock = threading.Lock() + self._recognizer: Optional[object] = None # cv2.face.LBPHFaceRecognizer + self._label_map: dict[int, str] = {} # int label → guest name + + self._stop_event = threading.Event() + self._thread: Optional[threading.Thread] = None + + # YuNet face detector — initialized in start() + self._detector: Optional[object] = None # cv2.FaceDetectorYN + self._detector_input_size: tuple = (0, 0) + + # ------------------------------------------------------------------ + # Public read properties (thread-safe) + # ------------------------------------------------------------------ + + @property + def current_name(self) -> str: + with self._lock: + return self._current_name + + @property + def current_encoding(self) -> Optional[np.ndarray]: + """Returns the latest detected face crop (grayscale numpy array) or None.""" + with self._lock: + return self._current_encoding.copy() if self._current_encoding is not None else None + + @property + def latest_annotated_jpeg(self) -> Optional[bytes]: + with self._lock: + return self._latest_annotated_jpeg + + @property + def confidence(self) -> float: + with self._lock: + return self._confidence + + def get_recent_logs(self, n: int = 50) -> list[str]: + with self._log_lock: + return list(self._log_buffer[-n:]) + + def set_face_event_callback(self, callback: Optional[Callable[[dict[str, Any]], None]]) -> None: + """Register a callback for stable face-state transition events.""" + with self._lock: + self._face_event_callback = callback + + def best_recent_face( + self, + window_seconds: float = _BEST_FACE_WINDOW_SECONDS, + require_dwell: bool = True, + ) -> tuple[str, float, Optional[np.ndarray]]: + """Return the best face seen in the last `window_seconds` seconds. + + Selection strategy: + - Among all entries in the rolling detection window (entries with + blur_score < _MIN_BLUR_SCORE are never stored), pick the one with the + highest combined quality score: blur_score * log(face_area) + This ranks sharpness first while using face area as a tiebreaker so + a large sharp face beats both a tiny sharp face and a large blurry one. + - Run LBPH recognition on that crop and return the result. + - If the window is empty (all recent crops were too blurry, or no face + seen) → return ("Unknown", 0.0, None). + + This is intentionally called at tool-invocation time (not in the + background loop) so the LLM always queries the best available evidence + rather than a momentary snapshot. + + Returns: + (name, lbph_confidence, crop) + name — registered guest name or "Unknown" + lbph_confidence — LBPH distance (lower = more certain match; 0 if no guests) + crop — 100×100 grayscale numpy array, or None if window is empty + """ + now = time.monotonic() + with self._lock: + # Prune stale entries + while self._detection_window and (now - self._detection_window[0][0]) > window_seconds: + self._detection_window.popleft() + + if not self._detection_window: + return "Unknown", 0.0, None + + # Optional dwell guard: for tool-calls we prefer stable evidence; + # for dashboard preview we may choose best available immediately. + if require_dwell: + timestamps = [e[0] for e in self._detection_window] + if max(timestamps) - min(timestamps) < _MIN_DWELL_SECONDS: + return "Unknown", 0.0, None + + # Pick the entry with the best combined quality: sharpness × log(area) + best = max(self._detection_window, key=lambda e: e[2] * math.log(max(e[1], 1)) * max(e[4], 0.01)) + _ts, face_area, blur_score, crop, det_conf = best + crop_copy = crop.copy() + + # Run LBPH recognition outside the lock (can be slow) + name, conf = self._recognize(crop_copy) + logger.debug( + "best_recent_face: face_area=%d blur=%.1f det_conf=%.2f name=%s lbph_conf=%.1f", + face_area, blur_score, det_conf, name, conf, + ) + self._add_log(f"check_current_face → {name} (area={face_area}px², blur={blur_score:.1f}, det={det_conf:.2f}, lbph={conf:.1f})") + return name, conf, crop_copy + + # ------------------------------------------------------------------ + # Recognizer rebuild (called by register_guest tool after adding a guest) + # ------------------------------------------------------------------ + + def rebuild_recognizer(self) -> None: + """Rebuild the LBPH recognizer from the current guest database. + + Call this after registering or updating a guest so the worker + immediately starts recognising the new face. + """ + if not _LBPH_AVAILABLE: + return + guests = self._face_db.get_all_guests_with_crops() + if not guests: + with self._recognizer_lock: + self._recognizer = None + self._label_map = {} + self._add_log("Recognizer cleared (no guests)") + return + + label_map: dict[int, str] = {} + label_crops: list[tuple[int, np.ndarray]] = [] + for idx, (name, crop) in enumerate(guests): + label_map[idx] = name + label_crops.append((idx, crop)) + + recognizer = _build_lbph_recognizer(label_crops) + with self._recognizer_lock: + self._recognizer = recognizer + self._label_map = label_map + self._add_log(f"Recognizer rebuilt ({len(label_map)} guest(s))") + logger.info("LBPH recognizer rebuilt with %d guest(s)", len(label_map)) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def start(self) -> None: + # Initialize YuNet detector (downloads model on first run) + try: + model_path = _ensure_yunet_model() + self._detector = cv2.FaceDetectorYN.create( + model=str(model_path), + config="", + input_size=(640, 480), + score_threshold=_YUNET_SCORE_THRESHOLD, + nms_threshold=_YUNET_NMS_THRESHOLD, + top_k=_YUNET_TOP_K, + ) + self._detector_input_size = (640, 480) + logger.info("YuNet face detector initialized") + self._add_log("YuNet face detector initialized ✓") + except Exception as e: + logger.error("Failed to initialize YuNet detector: %s", e) + self._add_log(f"WARNING: YuNet detector failed to initialize: {e}") + + # Build recognizer from existing DB on startup + self.rebuild_recognizer() + self._stop_event.clear() + self._thread = threading.Thread(target=self._run, daemon=True, name="face-recognition-worker") + self._thread.start() + logger.info("FaceRecognitionWorker started (camera_worker=%s)", self._camera_worker) + + def stop(self) -> None: + self._stop_event.set() + if self._thread: + self._thread.join(timeout=3.0) + logger.info("FaceRecognitionWorker stopped") + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _add_log(self, msg: str) -> None: + ts = time.strftime("%H:%M:%S") + entry = f"[{ts}] {msg}" + with self._log_lock: + self._log_buffer.append(entry) + if len(self._log_buffer) > 200: + self._log_buffer = self._log_buffer[-200:] + + def _emit_face_state_event(self, event: dict[str, Any]) -> None: + """Best-effort callback dispatch for external face context events.""" + callback = None + with self._lock: + callback = self._face_event_callback + if callback is None: + return + try: + callback(event) + except Exception as e: + logger.warning("Failed to dispatch face state event: %s", e) + + def _update_stable_state( + self, + observed_state: str, + observed_name: str, + lbph_confidence: float, + det_confidence: float, + ) -> None: + """Promote observed state into stable state with dwell and cooldown guards.""" + now = time.monotonic() + + with self._lock: + # Stage candidate transitions first. + candidate_changed = ( + observed_state != self._candidate_state + or (observed_state == "known" and observed_name != self._candidate_name) + ) + if candidate_changed: + self._candidate_state = observed_state + self._candidate_name = observed_name + self._candidate_since = now + return + + if observed_state == "no_face": + required = _NO_FACE_CONFIRM_SECONDS + elif observed_state == "multiple": + required = _MULTIPLE_PEOPLE_CONFIRM_SECONDS + else: + required = _FACE_STATE_CONFIRM_SECONDS + if (now - self._candidate_since) < required: + return + + previous_state = self._stable_state + previous_name = self._stable_name + stable_changed = ( + observed_state != previous_state + or (observed_state == "known" and observed_name != previous_name) + ) + if not stable_changed: + return + + self._stable_state = observed_state + self._stable_name = observed_name + self._stable_since = now + + # Public current_* values follow the stable state, not instantaneous observations. + self._current_name = observed_name if observed_state == "known" else "Unknown" + self._confidence = 0.0 if observed_state == "no_face" else float(lbph_confidence) + + can_emit_event = (now - self._last_event_sent_at) >= _FACE_EVENT_COOLDOWN_SECONDS + if can_emit_event: + self._last_event_sent_at = now + + self._add_log( + "Stable face: %s(%s) -> %s(%s)" + % (previous_state, previous_name, observed_state, observed_name) + ) + + if not can_emit_event: + self._add_log("Face context event skipped (cooldown)") + return + + event_payload = { + "event": "face_state_changed", + "state": observed_state, + "name": observed_name if observed_state == "known" else None, + "previous_state": previous_state, + "previous_name": previous_name if previous_state == "known" else None, + "lbph_confidence": round(float(lbph_confidence), 2), + "detection_confidence": round(float(det_confidence), 3), + "timestamp": time.time(), + } + self._emit_face_state_event(event_payload) + + def _detect_faces(self, frame: np.ndarray) -> list[tuple[int, int, int, int, int, float]]: + """Return list of (x, y, w, h, area, det_confidence) sorted by area descending. + + Uses YuNet (cv2.FaceDetectorYN) which returns a real confidence score (0–1) + per bounding box. The detector input size is updated dynamically per frame. + """ + if self._detector is None: + return [] + + h, w = frame.shape[:2] + if (w, h) != self._detector_input_size: + self._detector.setInputSize((w, h)) + self._detector_input_size = (w, h) + + _, faces = self._detector.detect(frame) + if faces is None or len(faces) == 0: + return [] + + results = [ + (int(f[0]), int(f[1]), int(f[2]), int(f[3]), int(f[2] * f[3]), float(f[14])) + for f in faces + ] + results.sort(key=lambda r: r[4], reverse=True) + return results + + def _recognize(self, gray_crop: np.ndarray) -> tuple[str, float]: + """Return (name, confidence) for a face crop. confidence is LBPH distance (lower = better match).""" + with self._recognizer_lock: + recognizer = self._recognizer + label_map = self._label_map + + if recognizer is None or not label_map: + return "Unknown", 0.0 + + try: + resized = cv2.resize(gray_crop, (100, 100)) + label_int, confidence = recognizer.predict(resized) + name = label_map.get(label_int, "Unknown") + if confidence <= self._confidence_threshold: + return name, float(confidence) + else: + return "Unknown", float(confidence) + except Exception as e: + logger.debug("LBPH predict error: %s", e) + return "Unknown", 0.0 + + # ------------------------------------------------------------------ + # Main loop + # ------------------------------------------------------------------ + + def _run(self) -> None: + if self._camera_worker is None: + logger.warning("FaceRecognitionWorker: no camera_worker provided — face recognition disabled") + self._add_log("WARNING: No camera_worker — face recognition disabled") + return + + self._add_log("Camera worker attached ✓ (OpenCV Haar + LBPH, robot camera)") + + frame_count = 0 + last_face_rects: list[tuple[int, int, int, int]] = [] # (x, y, w, h) + last_face_labels: list[str] = [] + + try: + while not self._stop_event.is_set(): + frame = self._camera_worker.get_latest_frame() + if frame is None: + time.sleep(0.05) + continue + + frame_count += 1 + do_recognition = frame_count % self._process_every_n == 0 + + if do_recognition: + rects = self._detect_faces(frame) # sorted by area descending; (x,y,w,h,area,det_conf) + + # Filter to faces whose centre falls inside the central detection zone + fh_full, fw_full = frame.shape[:2] + _zx1 = fw_full * _ZONE_X_MARGIN + _zx2 = fw_full * (1 - _ZONE_X_MARGIN) + _zy1 = fh_full * _ZONE_Y_MARGIN + _zy2 = fh_full * (1 - _ZONE_Y_MARGIN) + rects = [ + (x, y, w, h, area, det_conf) + for x, y, w, h, area, det_conf in rects + if _zx1 <= (x + w / 2) <= _zx2 and _zy1 <= (y + h / 2) <= _zy2 + ] + + # Strip area/conf for annotation (keep (x,y,w,h) tuples) + last_face_rects = [(x, y, w, h) for x, y, w, h, _area, _conf in rects] + last_face_labels = [] + + multiple_in_zone = len(rects) >= _MULTIPLE_PEOPLE_THRESHOLD + if multiple_in_zone: + # Multiple people in the central zone — defer + # single-person identification until one comes forward. + self._update_stable_state("multiple", "Unknown", 0.0, 0.0) + + if rects: + # Best face = largest area (index 0 after sort) + px, py, pw, ph, face_area, det_conf = rects[0] + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + fh, fw = gray.shape[:2] + # Clamp bbox to frame bounds (YuNet can return coords outside the frame) + x1 = max(0, px) + y1 = max(0, py) + x2 = min(fw, px + pw) + y2 = min(fh, py + ph) + crop = gray[y1:y2, x1:x2] + if crop.size == 0: + last_face_labels.append("Face (bad crop)") + continue + resized_crop = cv2.resize(crop, (100, 100)) + + # Compute sharpness via Laplacian variance (higher = sharper) + blur_score = cv2.Laplacian(resized_crop, cv2.CV_64F).var() + + # Add every valid crop to the detection window; low blur crops + # are naturally ranked lower by the quality score. + now = time.monotonic() + with self._lock: + self._detection_window.append((now, face_area, blur_score, resized_crop.copy(), det_conf)) + # Prune entries older than the configured window + while self._detection_window and (now - self._detection_window[0][0]) > _BEST_FACE_WINDOW_SECONDS: + self._detection_window.popleft() + # Keep current_encoding pointing to the best-quality crop in the window + best_entry = max(self._detection_window, key=lambda e: e[2] * math.log(max(e[1], 1)) * max(e[4], 0.01)) + self._current_encoding = best_entry[3].copy() + + if blur_score < _MIN_BLUR_SCORE: + logger.debug("Face crop rejected: blur_score=%.1f < threshold=%.1f", blur_score, _MIN_BLUR_SCORE) + + # Run LBPH recognition on the current crop for live display + name, conf = self._recognize(resized_crop) + + # When 2+ faces are in the zone we already emitted + # "multiple"; skip the per-person state emission so + # we don't oscillate. + if not multiple_in_zone: + observed_state = "known" if name != "Unknown" else "unknown" + observed_name = name if observed_state == "known" else "Unknown" + self._update_stable_state(observed_state, observed_name, conf, det_conf) + + # Build labels for all detected faces + for i, (x, y, w, h, area, conf_i) in enumerate(rects): + if i == 0: + det_tag = f"det={det_conf:.2f}" + if name != "Unknown": + last_face_labels.append(f"Known: {name} (lbph={conf:.0f}) {det_tag}") + else: + last_face_labels.append(f"Unknown (lbph={conf:.0f}) {det_tag}") + else: + last_face_labels.append(f"Face (det={conf_i:.2f})") + else: + # Prune the window even when no face detected + now = time.monotonic() + with self._lock: + while self._detection_window and (now - self._detection_window[0][0]) > _BEST_FACE_WINDOW_SECONDS: + self._detection_window.popleft() + if not self._detection_window: + self._current_encoding = None + + self._update_stable_state("no_face", "No face", 0.0, 0.0) + + # Annotate frame and encode as JPEG + annotated = self._annotate_frame(frame, last_face_rects, last_face_labels) + _, jpeg = cv2.imencode(".jpg", annotated, [cv2.IMWRITE_JPEG_QUALITY, 70]) + with self._lock: + self._latest_annotated_jpeg = jpeg.tobytes() + + except Exception as e: + logger.exception("FaceRecognitionWorker crashed: %s", e) + self._add_log(f"CRASH: {e}") + finally: + logger.info("FaceRecognitionWorker: stopped") + + def _annotate_frame( + self, + frame: np.ndarray, + face_rects: list[tuple[int, int, int, int]], + labels: list[str], + ) -> np.ndarray: + """Draw bounding boxes and labels on the frame.""" + out = frame.copy() + + # Draw the active central detection zone + fh, fw = out.shape[:2] + zone_x1 = int(fw * _ZONE_X_MARGIN) + zone_y1 = int(fh * _ZONE_Y_MARGIN) + zone_x2 = int(fw * (1 - _ZONE_X_MARGIN)) + zone_y2 = int(fh * (1 - _ZONE_Y_MARGIN)) + cv2.rectangle(out, (zone_x1, zone_y1), (zone_x2, zone_y2), (0, 210, 210), 2) + cv2.putText(out, "Detection Zone", (zone_x1 + 4, zone_y1 + 18), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 210, 210), 1) + + for i, (x, y, w, h) in enumerate(face_rects): + label = labels[i] if i < len(labels) else "Face" + is_known = label.startswith("Known:") + color = (0, 200, 0) if is_known else (0, 100, 255) # green / orange + + cv2.rectangle(out, (x, y), (x + w, y + h), color, 2) + cv2.rectangle(out, (x, y + h - 28), (x + w, y + h), color, cv2.FILLED) + cv2.putText( + out, + label, + (x + 4, y + h - 8), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + (255, 255, 255), + 1, + ) + return out diff --git a/src/reachy_mini_receptionist/gemini_live.py b/src/reachy_mini_receptionist/gemini_live.py new file mode 100644 index 0000000000000000000000000000000000000000..7f0b26f443d30e152d0a1de1c2005470ad2ffb2a --- /dev/null +++ b/src/reachy_mini_receptionist/gemini_live.py @@ -0,0 +1,754 @@ +"""Gemini Live API handler — drop-in replacement for OpenaiRealtimeHandler. + +Same public surface as ``openai_realtime.OpenaiRealtimeHandler`` so the +rest of the app (main.py, console.py, headless_personality_ui.py) can +switch backends with one env var (``VOICE_BACKEND=gemini``) without +code changes. + +Audio I/O is bidirectional PCM via Gemini's Live websocket. Tool +calling, VAD, voice synthesis all delegated to Gemini. + +This module imports ``google.genai`` LAZILY inside ``start_up`` so the +absence of the SDK does not break OpenAI-backend installs. +""" +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import os +import threading +import time +import uuid +from datetime import datetime +from typing import Any, Final, Literal, Optional, Tuple + +import numpy as np +from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item, audio_to_int16 +from numpy.typing import NDArray +from scipy.signal import resample + +from reachy_mini_receptionist.config import config +from reachy_mini_receptionist.prompts import get_session_voice, get_session_instructions +from reachy_mini_receptionist.tools.core_tools import ( + ToolDependencies, + get_tool_specs, +) +from reachy_mini_receptionist.tools.background_tool_manager import ( + ToolCallRoutine, + ToolNotification, + BackgroundToolManager, +) + +logger = logging.getLogger(__name__) + +# Gemini Live expects 16 kHz PCM16 mono input; outputs 24 kHz PCM16. +GEMINI_INPUT_SAMPLE_RATE: Final[int] = 16000 +GEMINI_OUTPUT_SAMPLE_RATE: Final[int] = 24000 + + +def _gemini_model_name() -> str: + """Resolve the Gemini Live model id (env-overridable). + + Default is ``gemini-2.5-flash-native-audio-latest`` — confirmed + bidiGenerateContent-capable on the project's API key (queried + 2026-05-20 against models.list endpoint). Set GEMINI_LIVE_MODEL + in .env to switch to e.g. gemini-2.5-flash-native-audio-preview-09-2025 + or whatever else your project has allowlisted. + """ + return (os.getenv("GEMINI_LIVE_MODEL") or "gemini-2.5-flash-native-audio-latest").strip() + + +def _openai_tools_to_gemini(openai_tools: list[dict]) -> list[dict]: + """Translate OpenAI function-tool specs to Gemini function declarations. + + OpenAI format: {"type": "function", "name": "...", "description": "...", + "parameters": {"type": "object", ...}} + Gemini format: {"function_declarations": [ + {"name": "...", "description": "...", + "parameters": {"type": "OBJECT", ...}}, ...]} + """ + declarations: list[dict] = [] + for tool in openai_tools or []: + if tool.get("type") != "function": + continue + name = tool.get("name") + if not name: + continue + decl = { + "name": name, + "description": tool.get("description", ""), + } + params = tool.get("parameters") + if params: + decl["parameters"] = params + declarations.append(decl) + return [{"function_declarations": declarations}] if declarations else [] + + +class GeminiLiveHandler(AsyncStreamHandler): + """Gemini Live API handler — mirror of OpenaiRealtimeHandler. + + Same public surface as the OpenAI handler so the rest of the app + can switch backends via ``VOICE_BACKEND=gemini`` without code + changes elsewhere. + """ + + def __init__( + self, + deps: ToolDependencies, + gradio_mode: bool = False, + instance_path: Optional[str] = None, + session_manager: Any | None = None, + controller: Any | None = None, + ): + super().__init__( + expected_layout="mono", + output_sample_rate=GEMINI_OUTPUT_SAMPLE_RATE, + input_sample_rate=GEMINI_INPUT_SAMPLE_RATE, + ) + self.deps = deps + self.gradio_mode = gradio_mode + self.instance_path = instance_path + self._session_manager = session_manager + self._controller = controller + + self.output_queue: "asyncio.Queue[Tuple[int, NDArray[np.int16]] | AdditionalOutputs]" = asyncio.Queue() + + self.session: Any = None # google.genai live session, set in start_up + self._client: Any = None + self._runtime_loop: asyncio.AbstractEventLoop | None = None + self._shutdown_requested: bool = False + + # Cumulative cost tracker (Gemini doesn't expose token costs the same way) + self.cumulative_cost: float = 0.0 + self.start_time = asyncio.get_event_loop().time() + self.last_activity_time = self.start_time + + # Background tool manager (same as OpenAI handler) + self.tool_manager = BackgroundToolManager() + + # Tool-call args stash keyed by call_id, so the controller can see + # both args + result when the tool completes. + self._tool_call_args: dict[str, dict[str, Any]] = {} + + # Last face event + session event sent to model (for /api endpoints) + self._face_event_lock = threading.Lock() + self._last_face_event_sent: dict[str, Any] | None = None + self._session_event_lock = threading.Lock() + self._last_session_event_sent: dict[str, Any] | None = None + + # Pending events waiting for session to be ready + self._pending_face_event_lock = threading.Lock() + self._pending_face_event: dict[str, Any] | None = None + self._pending_session_event_lock = threading.Lock() + self._pending_session_event: dict[str, Any] | None = None + + # idle-speech cue dedupe + self._idle_speech_cue_pushed: bool = False + + # ------------------------------------------------------------------ + # AsyncStreamHandler required methods + # ------------------------------------------------------------------ + + def copy(self) -> "GeminiLiveHandler": + return GeminiLiveHandler( + self.deps, + self.gradio_mode, + self.instance_path, + session_manager=self._session_manager, + controller=self._controller, + ) + + async def start_up(self) -> None: + """Connect to Gemini Live and run the event loop until shutdown.""" + self._runtime_loop = asyncio.get_running_loop() + + # Lazy import — only required when this backend is active. + try: + from google import genai + from google.genai import types as genai_types + except ImportError as e: + logger.error( + "google-genai SDK not installed. Install with: " + "pip install google-genai. Error: %s", e, + ) + return + + api_key = (os.getenv("GEMINI_API_KEY") or "").strip() + if not api_key: + logger.error("GEMINI_API_KEY not set — cannot start Gemini Live backend") + return + + self._client = genai.Client(api_key=api_key, http_options={"api_version": "v1beta"}) + model_id = _gemini_model_name() + logger.info("Gemini Live backend starting with model=%s", model_id) + + # Build session config — instructions, voice, tools, VAD-equivalent + instructions = get_session_instructions() + voice = get_session_voice() + openai_tools = get_tool_specs() # type: ignore[no-untyped-call] + gemini_tools = _openai_tools_to_gemini(openai_tools) + + # Minimal config — back to known-good shape after the + # realtime_input_config attempt caused 1011 internal server + # errors. Will revisit low-latency VAD tuning via SDK-specific + # types once we confirm the exact accepted shape. + config_obj: dict[str, Any] = { + "response_modalities": ["AUDIO"], + "system_instruction": instructions, + } + + # Half-cascade Live models (model name contains "-live-" but + # NOT "native-audio") require an explicit speech_config / + # voice_config to actually emit audio. Without it they degrade + # to 2-byte placeholder chunks and `resp.text`-only responses + # (observed 2026-05-21 with gemini-3.1-flash-live-preview). + # Native-audio models auto-select voice and REJECT voice_config + # with 1007 "Cannot extract voices from a non-audio request", + # so we conditionally apply this only when the model name says + # we should. Voice name override via env var. + model_lower = model_id.lower() + is_half_cascade = ("-live-" in model_lower or model_lower.endswith("-live-preview")) \ + and "native-audio" not in model_lower + if is_half_cascade: + voice_name = (os.getenv("GEMINI_LIVE_VOICE") or "Puck").strip() or "Puck" + config_obj["speech_config"] = { + "voice_config": { + "prebuilt_voice_config": {"voice_name": voice_name}, + }, + } + # Half-cascade also benefits from explicit input/output + # transcription so we can log what the model heard / said + # for debugging. Empty dict = enable with defaults. + config_obj["input_audio_transcription"] = {} + config_obj["output_audio_transcription"] = {} + logger.info( + "Gemini Live: half-cascade model detected — adding " + "speech_config (voice=%s) + transcription", voice_name, + ) + + if gemini_tools: + config_obj["tools"] = gemini_tools + + try: + async with self._client.aio.live.connect(model=model_id, config=config_obj) as session: + self.session = session + logger.info("Gemini Live connected.") + # Start background tool manager (same callback signature) + self.tool_manager.start_up(tool_callbacks=[self._handle_tool_result]) + + # Kick off conversation with a one-shot greeting prompt. + # With the tighter VAD config (400ms silence) we could + # in theory wait for visitor speech, but a proactive + # greeting keeps the demo natural — the bot says hi + # the moment a face is detected. Server VAD handles + # all subsequent visitor turns with low latency. + try: + await session.send_client_content( + turns=[{ + "role": "user", + "parts": [{ + "text": ( + "(Visitor just walked up. Greet them " + "very briefly in ONE short friendly " + "sentence and ask their name or who " + "they're here to see. Keep it under " + "8 words.)" + ), + }], + }], + turn_complete=True, + ) + logger.info("Gemini Live: sent kick-off greeting prompt") + except Exception as e: + logger.warning("Gemini Live: kick-off send failed: %s", e) + + try: + await self._run_event_loop() + except Exception as inner: + logger.exception("Gemini Live event loop crashed: %s", inner) + finally: + await self.tool_manager.shutdown() + self.session = None + logger.info("Gemini Live: session ended cleanly") + except Exception as e: + logger.exception("Gemini Live session failed: %s", e) + self.session = None + + async def shutdown(self) -> None: + self._shutdown_requested = True + try: + if self.session is not None: + await self.session.close() + except Exception as e: + logger.debug("Gemini session close ignored: %s", e) + try: + await self.tool_manager.shutdown() + except Exception: + pass + + async def receive(self, frame: Tuple[int, NDArray[np.int16]]) -> None: + """Stream visitor mic frames to Gemini Live.""" + if self.session is None: + return + input_rate, audio_frame = frame + if audio_frame.ndim == 2: + if audio_frame.shape[1] > audio_frame.shape[0]: + audio_frame = audio_frame.T + if audio_frame.shape[1] > 1: + audio_frame = audio_frame[:, 0] + # Resample if needed + if self.input_sample_rate != input_rate: + audio_frame = resample( + audio_frame, + int(len(audio_frame) * self.input_sample_rate / input_rate), + ) + audio_frame = audio_to_int16(audio_frame) + try: + await self.session.send_realtime_input( + audio={ + "data": audio_frame.tobytes(), + "mime_type": f"audio/pcm;rate={self.input_sample_rate}", + }, + ) + # Log once after first frame so we know mic-to-Gemini path is live. + if not getattr(self, "_first_mic_frame_logged", False): + logger.info( + "Gemini Live: first mic frame sent (input_rate=%d, target_rate=%d, samples=%d)", + input_rate, self.input_sample_rate, len(audio_frame), + ) + self._first_mic_frame_logged = True + except Exception as e: + logger.debug("Dropped mic frame: %s", e) + + async def emit(self) -> Tuple[int, NDArray[np.int16]] | AdditionalOutputs | None: + # Optional idle/timeout reset like OpenAI handler + if self._session_manager is not None: + try: + self._session_manager.maybe_reset_if_stale() + except Exception: + pass + return await wait_for_item(self.output_queue) # type: ignore[no-any-return] + + # ------------------------------------------------------------------ + # Event loop — read Gemini events and dispatch + # ------------------------------------------------------------------ + + async def _run_event_loop(self) -> None: + """Consume Gemini Live server events until session closes / shutdown. + + Wraps session.receive() in an outer loop so the conversation + survives multiple turns. Gemini's receive() iterator can return + after a single turn in some SDK versions — when it exits we + re-enter as long as the session and shutdown flag say keep going. + """ + if self.session is None: + return + event_count = 0 + audio_chunks = 0 + outer_iterations = 0 + try: + while self.session is not None and not self._shutdown_requested: + outer_iterations += 1 + logger.info("Gemini Live: receive() iteration %d starting", outer_iterations) + async for resp in self.session.receive(): + if self._shutdown_requested: + break + event_count += 1 + + # Smoke test confirmed: resp.data IS the same audio + # as server_content.model_turn.parts[].inline_data.data + # (just a convenience shortcut). Use ONLY the shortcut + # to avoid double-emit when both paths fire. + data = getattr(resp, "data", None) + if data: + audio_chunks += 1 + arr = np.frombuffer(data, dtype=np.int16).reshape(1, -1) + await self.output_queue.put((self.output_sample_rate, arr)) + self.last_activity_time = asyncio.get_event_loop().time() + if audio_chunks == 1: + logger.info("Gemini Live: first audio chunk received (%d bytes)", len(data)) + + # text shortcut — concatenated text parts + text = getattr(resp, "text", None) + if text: + logger.info("Gemini Live text: %r", text[:120]) + await self.output_queue.put( + AdditionalOutputs({"role": "assistant", "content": text}) + ) + + # server_content parsing for non-audio metadata + turn_complete + server_content = getattr(resp, "server_content", None) + if server_content is not None: + # NOTE: audio chunks via model_turn.parts[].inline_data + # are ALREADY surfaced via resp.data above — don't + # re-emit. Just look at non-audio parts here. + model_turn = getattr(server_content, "model_turn", None) + if model_turn is not None: + parts = getattr(model_turn, "parts", None) or [] + for part in parts: + if getattr(part, "thought", False): + continue # silently swallow chain-of-thought + ptext = getattr(part, "text", None) + if ptext and not text: # avoid double-text-emit + logger.info("Gemini Live model_turn text: %r", ptext[:120]) + await self.output_queue.put( + AdditionalOutputs({"role": "assistant", "content": ptext}) + ) + + in_tr = getattr(server_content, "input_transcription", None) + if in_tr is not None: + txt = (getattr(in_tr, "text", "") or "").strip() + if txt: + logger.info("Gemini Live input transcript: %r", txt) + if self._session_manager is not None: + try: + self._session_manager.record_user_transcript(txt) + except Exception: + pass + await self.output_queue.put( + AdditionalOutputs({"role": "user", "content": txt}) + ) + + if getattr(server_content, "turn_complete", False): + logger.info( + "Gemini Live: model turn complete (events=%d, audio_chunks=%d)", + event_count, audio_chunks, + ) + audio_chunks = 0 # reset for next turn + + # Tool calls + tool_call = getattr(resp, "tool_call", None) + if tool_call is not None: + fcs = getattr(tool_call, "function_calls", None) or [] + logger.info("Gemini Live: tool_call with %d function calls", len(fcs)) + for fc in fcs: + await self._dispatch_function_call(fc) + + logger.info( + "Gemini Live: receive() iteration %d ended (events so far=%d). Looping.", + outer_iterations, event_count, + ) + # Small backoff so we don't spin if session is genuinely dead + await asyncio.sleep(0.1) + except Exception as e: + logger.warning("Gemini Live event loop exited with exception: %s", e) + finally: + logger.info( + "Gemini Live event loop: total iterations=%d, events=%d, audio_chunks=%d", + outer_iterations, event_count, audio_chunks, + ) + + async def _dispatch_function_call(self, fc: Any) -> None: + """Route a Gemini function call into the background tool manager.""" + tool_name = getattr(fc, "name", None) + args_obj = getattr(fc, "args", {}) or {} + call_id = str(getattr(fc, "id", None) or uuid.uuid4()) + if not tool_name: + return + # Normalize args dict + if not isinstance(args_obj, dict): + try: + args_obj = dict(args_obj) + except Exception: + args_obj = {} + args_json_str = json.dumps(args_obj) + self._tool_call_args[call_id] = args_obj + logger.info( + "Gemini tool call: %s call_id=%s args=%s", tool_name, call_id, args_json_str, + ) + try: + await self.tool_manager.start_tool( + call_id=call_id, + tool_call_routine=ToolCallRoutine( + tool_name=tool_name, + args_json_str=args_json_str, + deps=self.deps, + ), + is_idle_tool_call=False, + ) + except Exception as e: + logger.warning("Failed to start Gemini tool '%s': %s", tool_name, e) + + async def _handle_tool_result(self, bg_tool: ToolNotification) -> None: + """Send the tool result back to Gemini + notify the controller.""" + if bg_tool.error is not None: + tool_result: dict[str, Any] = {"error": bg_tool.error} + elif bg_tool.result is not None: + tool_result = bg_tool.result + else: + tool_result = {"error": "No result"} + + call_args = self._tool_call_args.pop(bg_tool.id, {}) + + # Send result back to Gemini Live + if self.session is not None: + try: + # Gemini expects function_responses with {id, name, response: {output: ...}} + await self.session.send_tool_response( + function_responses=[{ + "id": bg_tool.id if isinstance(bg_tool.id, str) else None, + "name": bg_tool.tool_name, + "response": {"output": tool_result}, + }], + ) + except Exception as e: + logger.debug("send_tool_response failed: %s", e) + + # Surface tool result to dashboard chatbot + await self.output_queue.put( + AdditionalOutputs({ + "role": "assistant", + "content": json.dumps(tool_result), + "metadata": { + "title": f"🛠️ Used tool {bg_tool.tool_name}", + "status": "done", + }, + }) + ) + + # Drive backend state transitions (same as OpenAI handler) + if self._controller is not None: + try: + await self._controller.on_tool_completed_async( + bg_tool.tool_name, call_args, tool_result, + ) + except Exception as e: + logger.warning( + "ConversationController.on_tool_completed_async raised %s: %s", + type(e).__name__, e, + ) + + # ------------------------------------------------------------------ + # Public API — face + session context push (mirror of OpenAI handler) + # ------------------------------------------------------------------ + + def _stash_pending_face_event(self, face_event: dict[str, Any]) -> None: + with self._pending_face_event_lock: + self._pending_face_event = dict(face_event) + + def _pop_pending_face_event(self) -> dict[str, Any] | None: + with self._pending_face_event_lock: + p = self._pending_face_event + self._pending_face_event = None + return p + + async def _flush_pending_face_event(self) -> None: + p = self._pop_pending_face_event() + if p is not None: + try: + await self._push_face_context_event(p) + except Exception as e: + logger.debug("flush pending face event failed: %s", e) + self._stash_pending_face_event(p) + + def notify_external_face_event(self, face_event: dict[str, Any]) -> None: + loop = self._runtime_loop + if loop is None or loop.is_closed() or self.session is None: + self._stash_pending_face_event(face_event) + return + try: + future = asyncio.run_coroutine_threadsafe( + self._push_face_context_event(face_event), loop, + ) + + def _done(fut: "asyncio.Future[None]") -> None: + try: + fut.result() + except Exception as e: + logger.debug("face event push failed: %s", e) + self._stash_pending_face_event(face_event) + + future.add_done_callback(_done) + except Exception as e: + logger.debug("schedule face event failed: %s", e) + self._stash_pending_face_event(face_event) + + async def _push_face_context_event(self, face_event: dict[str, Any]) -> None: + if self.session is None: + self._stash_pending_face_event(face_event) + return + state = str(face_event.get("state", "unknown")) + name = face_event.get("name") + msg = ( + f"[External face update {self.format_timestamp()}] " + f"state={state}; name={name}. Context only; don't respond unless the user speaks." + ) + try: + # send_realtime_input(text=...) injects context without + # forcing a turn — perfect for face state updates that + # the model should know about but not respond to. + await self.session.send_realtime_input(text=msg) + except Exception as e: + logger.debug("send face context failed: %s", e) + self._stash_pending_face_event(face_event) + return + + sent_at = time.time() + payload = { + "state": state, + "name": name, + "previous_state": face_event.get("previous_state"), + "previous_name": face_event.get("previous_name"), + "lbph_confidence": float(face_event.get("lbph_confidence") or 0.0), + "detection_confidence": float(face_event.get("detection_confidence") or 0.0), + "sent_at": sent_at, + "sent_at_iso": datetime.fromtimestamp(sent_at).strftime("%Y-%m-%d %H:%M:%S"), + } + with self._face_event_lock: + self._last_face_event_sent = payload + + def get_last_face_event_sent(self) -> dict[str, Any] | None: + with self._face_event_lock: + return dict(self._last_face_event_sent) if self._last_face_event_sent else None + + # --- session events --- + + def _stash_pending_session_event(self, payload: dict[str, Any]) -> None: + with self._pending_session_event_lock: + self._pending_session_event = dict(payload) + + def _pop_pending_session_event(self) -> dict[str, Any] | None: + with self._pending_session_event_lock: + p = self._pending_session_event + self._pending_session_event = None + return p + + async def _flush_pending_session_event(self) -> None: + p = self._pop_pending_session_event() + if p is not None: + try: + await self._push_session_context_event(p) + except Exception: + self._stash_pending_session_event(p) + + def notify_session_event(self, previous_state: Any, new_state: Any, snapshot: Any) -> None: + try: + payload = { + "previous_state": getattr(previous_state, "value", str(previous_state)), + "new_state": getattr(new_state, "value", str(new_state)), + "snapshot": snapshot.to_dict() if hasattr(snapshot, "to_dict") else {}, + } + except Exception: + return + + if payload.get("new_state") == "idle": + self._idle_speech_cue_pushed = False + + loop = self._runtime_loop + if loop is None or loop.is_closed() or self.session is None: + self._stash_pending_session_event(payload) + return + try: + future = asyncio.run_coroutine_threadsafe( + self._push_session_context_event(payload), loop, + ) + + def _done(fut: "asyncio.Future[None]") -> None: + try: + fut.result() + except Exception: + self._stash_pending_session_event(payload) + + future.add_done_callback(_done) + except Exception: + self._stash_pending_session_event(payload) + + async def _push_session_context_event(self, payload: dict[str, Any]) -> None: + if self.session is None: + self._stash_pending_session_event(payload) + return + snap = payload.get("snapshot") or {} + new_state_value = payload.get("new_state") + hint = "" + speak_now = False + try: + from reachy_mini_receptionist.conversation_controller import ( + next_action_hint, should_speak_immediately, + ) + from reachy_mini_receptionist.receptionist_state import ReceptionState + if new_state_value: + new_state_enum = ReceptionState(new_state_value) + hint = next_action_hint(new_state_enum) + speak_now = should_speak_immediately(new_state_enum) + except Exception: + pass + + base = ( + f"[Backend session update {self.format_timestamp()}] " + f"state: {payload.get('previous_state')} -> {new_state_value}; " + f"visitor={snap.get('visitor_name')}; " + f"employee={snap.get('employee_name')}; " + f"appointment={(snap.get('matched_appointment') or {}).get('time')}; " + f"email_sent_to={snap.get('email_sent_to')}." + ) + if hint and speak_now: + msg = f"{base} SPEAK NOW: {hint}" + elif hint: + msg = f"{base} Next: {hint} (Stay quiet until the visitor speaks; context only.)" + else: + msg = f"{base} Context only; do not respond unless the user speaks." + + try: + # For SPEAK_NOW transitions, use send_client_content with + # turn_complete=True so the model actually responds. + # Otherwise context-only via send_realtime_input(text=). + from reachy_mini_receptionist.conversation_controller import should_speak_immediately + from reachy_mini_receptionist.receptionist_state import ReceptionState + speak_now = False + try: + if new_state_value: + speak_now = should_speak_immediately(ReceptionState(new_state_value)) + except Exception: + pass + + if speak_now: + await self.session.send_client_content( + turns=[{ + "role": "user", + "parts": [{"text": msg}], + }], + turn_complete=True, + ) + else: + await self.session.send_realtime_input(text=msg) + except Exception as e: + logger.debug("session context push failed: %s", e) + return + + sent_payload = {**payload, "sent_at": time.time(), "hint": hint} + with self._session_event_lock: + self._last_session_event_sent = sent_payload + + def get_last_session_event_sent(self) -> dict[str, Any] | None: + with self._session_event_lock: + return dict(self._last_session_event_sent) if self._last_session_event_sent else None + + # ------------------------------------------------------------------ + # Personality + voice (UI hooks) + # ------------------------------------------------------------------ + + async def apply_personality(self, profile: str | None) -> str: + """Profile updates require a session restart — minimal impl.""" + try: + from reachy_mini_receptionist.config import set_custom_profile + set_custom_profile(profile) + return "Personality updated. Restart for it to take effect (Gemini backend)." + except Exception as e: + return f"Failed to apply personality: {e}" + + async def get_available_voices(self) -> list[str]: + """Gemini prebuilt voices (fixed list — no discovery API).""" + return ["Aoede", "Charon", "Kore", "Puck", "Fenrir"] + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def format_timestamp(self) -> str: + loop_time = asyncio.get_event_loop().time() + elapsed = loop_time - self.start_time + dt = datetime.now() + return f"[{dt.strftime('%Y-%m-%d %H:%M:%S')} | +{elapsed:.1f}s]" diff --git a/src/reachy_mini_receptionist/gradio_personality.py b/src/reachy_mini_receptionist/gradio_personality.py new file mode 100644 index 0000000000000000000000000000000000000000..5d09c8311e89d6be75dd7431fd3f130924e46d66 --- /dev/null +++ b/src/reachy_mini_receptionist/gradio_personality.py @@ -0,0 +1,316 @@ +"""Gradio personality UI components and wiring. + +This module encapsulates the UI elements and logic related to managing +conversation "personalities" (profiles) so that `main.py` stays lean. +""" + +from __future__ import annotations +from typing import Any +from pathlib import Path + +import gradio as gr + +from .config import LOCKED_PROFILE, config + + +class PersonalityUI: + """Container for personality-related Gradio components.""" + + def __init__(self) -> None: + """Initialize the PersonalityUI instance.""" + # Constants and paths + self.DEFAULT_OPTION = "(built-in default)" + self._profiles_root = Path(__file__).parent / "profiles" + self._tools_dir = Path(__file__).parent / "tools" + self._prompts_dir = Path(__file__).parent / "prompts" + + # Components (initialized in create_components) + self.personalities_dropdown: gr.Dropdown + self.apply_btn: gr.Button + self.status_md: gr.Markdown + self.preview_md: gr.Markdown + self.person_name_tb: gr.Textbox + self.person_instr_ta: gr.TextArea + self.tools_txt_ta: gr.TextArea + self.voice_dropdown: gr.Dropdown + self.new_personality_btn: gr.Button + self.available_tools_cg: gr.CheckboxGroup + self.save_btn: gr.Button + + # ---------- Filesystem helpers ---------- + def _list_personalities(self) -> list[str]: + names: list[str] = [] + try: + if self._profiles_root.exists(): + for p in sorted(self._profiles_root.iterdir()): + if p.name == "user_personalities": + continue + if p.is_dir() and (p / "instructions.txt").exists(): + names.append(p.name) + user_dir = self._profiles_root / "user_personalities" + if user_dir.exists(): + for p in sorted(user_dir.iterdir()): + if p.is_dir() and (p / "instructions.txt").exists(): + names.append(f"user_personalities/{p.name}") + except Exception: + pass + return names + + def _resolve_profile_dir(self, selection: str) -> Path: + return self._profiles_root / selection + + def _read_instructions_for(self, name: str) -> str: + try: + if name == self.DEFAULT_OPTION: + default_file = self._prompts_dir / "default_prompt.txt" + if default_file.exists(): + return default_file.read_text(encoding="utf-8").strip() + return "" + target = self._resolve_profile_dir(name) / "instructions.txt" + if target.exists(): + return target.read_text(encoding="utf-8").strip() + return "" + except Exception as e: + return f"Could not load instructions: {e}" + + @staticmethod + def _sanitize_name(name: str) -> str: + import re + + s = name.strip() + s = re.sub(r"\s+", "_", s) + s = re.sub(r"[^a-zA-Z0-9_-]", "", s) + return s + + # ---------- Public API ---------- + def create_components(self) -> None: + """Instantiate Gradio components for the personality UI.""" + if LOCKED_PROFILE is not None: + is_locked = True + current_value: str = LOCKED_PROFILE + dropdown_label = "Select personality (locked)" + dropdown_choices: list[str] = [LOCKED_PROFILE] + else: + is_locked = False + current_value = config.REACHY_MINI_CUSTOM_PROFILE or self.DEFAULT_OPTION + dropdown_label = "Select personality" + dropdown_choices = [self.DEFAULT_OPTION, *(self._list_personalities())] + + self.personalities_dropdown = gr.Dropdown( + label=dropdown_label, + choices=dropdown_choices, + value=current_value, + interactive=not is_locked, + ) + self.apply_btn = gr.Button("Apply personality", interactive=not is_locked) + self.status_md = gr.Markdown(visible=True) + self.preview_md = gr.Markdown(value=self._read_instructions_for(current_value)) + self.person_name_tb = gr.Textbox(label="Personality name", interactive=not is_locked) + self.person_instr_ta = gr.TextArea(label="Personality instructions", lines=10, interactive=not is_locked) + self.tools_txt_ta = gr.TextArea(label="tools.txt", lines=10, interactive=not is_locked) + self.voice_dropdown = gr.Dropdown(label="Voice", choices=["marin"], value="marin", interactive=not is_locked) + self.new_personality_btn = gr.Button("New personality", interactive=not is_locked) + self.available_tools_cg = gr.CheckboxGroup(label="Available tools (helper)", choices=[], value=[], interactive=not is_locked) + self.save_btn = gr.Button("Save personality (instructions + tools)", interactive=not is_locked) + + def additional_inputs_ordered(self) -> list[Any]: + """Return the additional inputs in the expected order for Stream.""" + return [ + self.personalities_dropdown, + self.apply_btn, + self.new_personality_btn, + self.status_md, + self.preview_md, + self.person_name_tb, + self.person_instr_ta, + self.tools_txt_ta, + self.voice_dropdown, + self.available_tools_cg, + self.save_btn, + ] + + # ---------- Event wiring ---------- + def wire_events(self, handler: Any, blocks: gr.Blocks) -> None: + """Attach event handlers to components within a Blocks context.""" + + async def _apply_personality(selected: str) -> tuple[str, str]: + if LOCKED_PROFILE is not None and selected != LOCKED_PROFILE: + return ( + f"Profile is locked to '{LOCKED_PROFILE}'. Cannot change personality.", + self._read_instructions_for(LOCKED_PROFILE), + ) + profile = None if selected == self.DEFAULT_OPTION else selected + status = await handler.apply_personality(profile) + preview = self._read_instructions_for(selected) + return status, preview + + def _read_voice_for(name: str) -> str: + try: + if name == self.DEFAULT_OPTION: + return "marin" + vf = self._resolve_profile_dir(name) / "voice.txt" + if vf.exists(): + v = vf.read_text(encoding="utf-8").strip() + return v or "marin" + except Exception: + pass + return "marin" + + async def _fetch_voices(selected: str) -> dict[str, Any]: + try: + voices = await handler.get_available_voices() + current = _read_voice_for(selected) + if current not in voices: + current = "marin" + return gr.update(choices=voices, value=current) + except Exception: + return gr.update(choices=["marin"], value="marin") + + def _available_tools_for(selected: str) -> tuple[list[str], list[str]]: + shared: list[str] = [] + try: + for py in self._tools_dir.glob("*.py"): + if py.stem in {"__init__", "core_tools"}: + continue + shared.append(py.stem) + except Exception: + pass + local: list[str] = [] + try: + if selected != self.DEFAULT_OPTION: + for py in (self._profiles_root / selected).glob("*.py"): + local.append(py.stem) + except Exception: + pass + return sorted(shared), sorted(local) + + def _parse_enabled_tools(text: str) -> list[str]: + enabled: list[str] = [] + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + enabled.append(s) + return enabled + + def _load_profile_for_edit(selected: str) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], str]: + instr = self._read_instructions_for(selected) + tools_txt = "" + if selected != self.DEFAULT_OPTION: + tp = self._resolve_profile_dir(selected) / "tools.txt" + if tp.exists(): + tools_txt = tp.read_text(encoding="utf-8") + shared, local = _available_tools_for(selected) + all_tools = sorted(set(shared + local)) + enabled = _parse_enabled_tools(tools_txt) + status_text = f"Loaded profile '{selected}'." + return ( + gr.update(value=instr), + gr.update(value=tools_txt), + gr.update(choices=all_tools, value=enabled), + status_text, + ) + + def _new_personality() -> tuple[ + dict[str, Any], dict[str, Any], dict[str, Any], dict[str, Any], str, dict[str, Any] + ]: + try: + # Prefill with hints + instr_val = """# Write your instructions here\n# e.g., Keep responses concise and friendly.""" + tools_txt_val = "# tools enabled for this profile\n" + return ( + gr.update(value=""), + gr.update(value=instr_val), + gr.update(value=tools_txt_val), + gr.update(choices=sorted(_available_tools_for(self.DEFAULT_OPTION)[0]), value=[]), + "Fill in a name, instructions and (optional) tools, then Save.", + gr.update(value="marin"), + ) + except Exception: + return ( + gr.update(), + gr.update(), + gr.update(), + gr.update(), + "Failed to initialize new personality.", + gr.update(), + ) + + def _save_personality( + name: str, instructions: str, tools_text: str, voice: str + ) -> tuple[dict[str, Any], dict[str, Any], str]: + name_s = self._sanitize_name(name) + if not name_s: + return gr.update(), gr.update(), "Please enter a valid name." + try: + target_dir = self._profiles_root / "user_personalities" / name_s + target_dir.mkdir(parents=True, exist_ok=True) + (target_dir / "instructions.txt").write_text(instructions.strip() + "\n", encoding="utf-8") + (target_dir / "tools.txt").write_text(tools_text.strip() + "\n", encoding="utf-8") + (target_dir / "voice.txt").write_text((voice or "marin").strip() + "\n", encoding="utf-8") + + choices = self._list_personalities() + value = f"user_personalities/{name_s}" + if value not in choices: + choices.append(value) + return ( + gr.update(choices=[self.DEFAULT_OPTION, *sorted(choices)], value=value), + gr.update(value=instructions), + f"Saved personality '{name_s}'.", + ) + except Exception as e: + return gr.update(), gr.update(), f"Failed to save personality: {e}" + + def _sync_tools_from_checks(selected: list[str], current_text: str) -> dict[str, Any]: + comments = [ln for ln in current_text.splitlines() if ln.strip().startswith("#")] + body = "\n".join(selected) + out = ("\n".join(comments) + ("\n" if comments else "") + body).strip() + "\n" + return gr.update(value=out) + + with blocks: + self.apply_btn.click( + fn=_apply_personality, + inputs=[self.personalities_dropdown], + outputs=[self.status_md, self.preview_md], + ) + + self.personalities_dropdown.change( + fn=_load_profile_for_edit, + inputs=[self.personalities_dropdown], + outputs=[self.person_instr_ta, self.tools_txt_ta, self.available_tools_cg, self.status_md], + ) + + blocks.load( + fn=_fetch_voices, + inputs=[self.personalities_dropdown], + outputs=[self.voice_dropdown], + ) + + self.available_tools_cg.change( + fn=_sync_tools_from_checks, + inputs=[self.available_tools_cg, self.tools_txt_ta], + outputs=[self.tools_txt_ta], + ) + + self.new_personality_btn.click( + fn=_new_personality, + inputs=[], + outputs=[ + self.person_name_tb, + self.person_instr_ta, + self.tools_txt_ta, + self.available_tools_cg, + self.status_md, + self.voice_dropdown, + ], + ) + + self.save_btn.click( + fn=_save_personality, + inputs=[self.person_name_tb, self.person_instr_ta, self.tools_txt_ta, self.voice_dropdown], + outputs=[self.personalities_dropdown, self.person_instr_ta, self.status_md], + ).then( + fn=_apply_personality, + inputs=[self.personalities_dropdown], + outputs=[self.status_md, self.preview_md], + ) diff --git a/src/reachy_mini_receptionist/headless_personality.py b/src/reachy_mini_receptionist/headless_personality.py new file mode 100644 index 0000000000000000000000000000000000000000..6df56b3ebe4af8319b25978fb9f88c08d53cb4d0 --- /dev/null +++ b/src/reachy_mini_receptionist/headless_personality.py @@ -0,0 +1,102 @@ +"""Headless personality management (console-based). + +Provides an interactive CLI to browse, preview, apply, create and edit +"personalities" (profiles) when running without Gradio. + +This module is intentionally not shared with the Gradio implementation to +avoid coupling and keep responsibilities clear for headless mode. +""" + +from __future__ import annotations +from typing import List +from pathlib import Path + + +DEFAULT_OPTION = "(built-in default)" + + +def _profiles_root() -> Path: + return Path(__file__).parent / "profiles" + + +def _prompts_dir() -> Path: + return Path(__file__).parent / "prompts" + + +def _tools_dir() -> Path: + return Path(__file__).parent / "tools" + + +def _sanitize_name(name: str) -> str: + import re + + s = name.strip() + s = re.sub(r"\s+", "_", s) + s = re.sub(r"[^a-zA-Z0-9_-]", "", s) + return s + + +def list_personalities() -> List[str]: + """List available personality profile names.""" + names: List[str] = [] + root = _profiles_root() + try: + if root.exists(): + for p in sorted(root.iterdir()): + if p.name == "user_personalities": + continue + if p.is_dir() and (p / "instructions.txt").exists(): + names.append(p.name) + udir = root / "user_personalities" + if udir.exists(): + for p in sorted(udir.iterdir()): + if p.is_dir() and (p / "instructions.txt").exists(): + names.append(f"user_personalities/{p.name}") + except Exception: + pass + return names + + +def resolve_profile_dir(selection: str) -> Path: + """Resolve the directory path for the given profile selection.""" + return _profiles_root() / selection + + +def read_instructions_for(name: str) -> str: + """Read the instructions.txt content for the given profile name.""" + try: + if name == DEFAULT_OPTION: + df = _prompts_dir() / "default_prompt.txt" + return df.read_text(encoding="utf-8").strip() if df.exists() else "" + target = resolve_profile_dir(name) / "instructions.txt" + return target.read_text(encoding="utf-8").strip() if target.exists() else "" + except Exception as e: + return f"Could not load instructions: {e}" + + +def available_tools_for(selected: str) -> List[str]: + """List available tool modules for the given profile selection.""" + shared: List[str] = [] + try: + for py in _tools_dir().glob("*.py"): + if py.stem in {"__init__", "core_tools"}: + continue + shared.append(py.stem) + except Exception: + pass + local: List[str] = [] + try: + if selected != DEFAULT_OPTION: + for py in resolve_profile_dir(selected).glob("*.py"): + local.append(py.stem) + except Exception: + pass + return sorted(set(shared + local)) + + +def _write_profile(name_s: str, instructions: str, tools_text: str, voice: str = "marin") -> None: + target_dir = _profiles_root() / "user_personalities" / name_s + target_dir.mkdir(parents=True, exist_ok=True) + (target_dir / "instructions.txt").write_text(instructions.strip() + "\n", encoding="utf-8") + (target_dir / "tools.txt").write_text((tools_text or "").strip() + "\n", encoding="utf-8") + (target_dir / "voice.txt").write_text((voice or "marin").strip() + "\n", encoding="utf-8") diff --git a/src/reachy_mini_receptionist/headless_personality_ui.py b/src/reachy_mini_receptionist/headless_personality_ui.py new file mode 100644 index 0000000000000000000000000000000000000000..0dbac553c5e8c32edfea6deef9c238d578e9cfa8 --- /dev/null +++ b/src/reachy_mini_receptionist/headless_personality_ui.py @@ -0,0 +1,287 @@ +"""Settings UI routes for headless personality management. + +Exposes REST endpoints on the provided FastAPI settings app. The +implementation schedules backend actions (apply personality, fetch voices) +onto the running LocalStream asyncio loop using the supplied get_loop +callable to avoid cross-thread issues. +""" + +from __future__ import annotations +import asyncio +import logging +from typing import Any, Callable, Optional + +from fastapi import FastAPI + +from .config import LOCKED_PROFILE, config +from .openai_realtime import OpenaiRealtimeHandler +from .headless_personality import ( + DEFAULT_OPTION, + _sanitize_name, + _write_profile, + list_personalities, + available_tools_for, + resolve_profile_dir, + read_instructions_for, +) + + +def mount_personality_routes( + app: FastAPI, + handler: OpenaiRealtimeHandler, + get_loop: Callable[[], asyncio.AbstractEventLoop | None], + *, + persist_personality: Callable[[Optional[str]], None] | None = None, + get_persisted_personality: Callable[[], Optional[str]] | None = None, +) -> None: + """Register personality management endpoints on a FastAPI app.""" + try: + from fastapi import Request + from pydantic import BaseModel + from fastapi.responses import JSONResponse + except Exception: # pragma: no cover - only when settings app not available + return + + class SavePayload(BaseModel): + name: str + instructions: str + tools_text: str + voice: Optional[str] = "marin" + + class ApplyPayload(BaseModel): + name: str + persist: Optional[bool] = False + + def _startup_choice() -> Any: + """Return the persisted startup personality or default.""" + try: + if get_persisted_personality is not None: + stored = get_persisted_personality() + if stored: + return stored + env_val = getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None) + if env_val: + return env_val + except Exception: + pass + return DEFAULT_OPTION + + def _current_choice() -> str: + try: + cur = getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None) + return cur or DEFAULT_OPTION + except Exception: + return DEFAULT_OPTION + + @app.get("/personalities") + def _list() -> dict: # type: ignore + choices = [DEFAULT_OPTION, *list_personalities()] + return { + "choices": choices, + "current": _current_choice(), + "startup": _startup_choice(), + "locked": LOCKED_PROFILE is not None, + "locked_to": LOCKED_PROFILE, + } + + @app.get("/personalities/load") + def _load(name: str) -> dict: # type: ignore + instr = read_instructions_for(name) + tools_txt = "" + voice = "marin" + if name != DEFAULT_OPTION: + pdir = resolve_profile_dir(name) + tp = pdir / "tools.txt" + if tp.exists(): + tools_txt = tp.read_text(encoding="utf-8") + vf = pdir / "voice.txt" + if vf.exists(): + v = vf.read_text(encoding="utf-8").strip() + voice = v or "marin" + avail = available_tools_for(name) + enabled = [ln.strip() for ln in tools_txt.splitlines() if ln.strip() and not ln.strip().startswith("#")] + return { + "instructions": instr, + "tools_text": tools_txt, + "voice": voice, + "available_tools": avail, + "enabled_tools": enabled, + } + + @app.post("/personalities/save") + async def _save(request: Request) -> dict: # type: ignore + # Accept raw JSON only to avoid validation-related 422s + try: + raw = await request.json() + except Exception: + raw = {} + name = str(raw.get("name", "")) + instructions = str(raw.get("instructions", "")) + tools_text = str(raw.get("tools_text", "")) + voice = str(raw.get("voice", "marin")) if raw.get("voice") is not None else "marin" + + name_s = _sanitize_name(name) + if not name_s: + return JSONResponse({"ok": False, "error": "invalid_name"}, status_code=400) # type: ignore + try: + logger.info( + "Headless save: name=%r voice=%r instr_len=%d tools_len=%d", + name_s, + voice, + len(instructions), + len(tools_text), + ) + _write_profile(name_s, instructions, tools_text, voice or "marin") + value = f"user_personalities/{name_s}" + choices = [DEFAULT_OPTION, *list_personalities()] + return {"ok": True, "value": value, "choices": choices} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) # type: ignore + + @app.post("/personalities/save_raw") + async def _save_raw( + request: Request, + name: Optional[str] = None, + instructions: Optional[str] = None, + tools_text: Optional[str] = None, + voice: Optional[str] = None, + ) -> dict: # type: ignore + # Accept query params, form-encoded, or raw JSON + data = {"name": name, "instructions": instructions, "tools_text": tools_text, "voice": voice} + # Prefer form if present + try: + form = await request.form() + for k in ("name", "instructions", "tools_text", "voice"): + if k in form and form[k] is not None: + data[k] = str(form[k]) + except Exception: + pass + # Try JSON + try: + raw = await request.json() + if isinstance(raw, dict): + for k in ("name", "instructions", "tools_text", "voice"): + if raw.get(k) is not None: + data[k] = str(raw.get(k)) + except Exception: + pass + + name_s = _sanitize_name(str(data.get("name") or "")) + if not name_s: + return JSONResponse({"ok": False, "error": "invalid_name"}, status_code=400) # type: ignore + instr = str(data.get("instructions") or "") + tools = str(data.get("tools_text") or "") + v = str(data.get("voice") or "marin") + try: + logger.info( + "Headless save_raw: name=%r voice=%r instr_len=%d tools_len=%d", name_s, v, len(instr), len(tools) + ) + _write_profile(name_s, instr, tools, v) + value = f"user_personalities/{name_s}" + choices = [DEFAULT_OPTION, *list_personalities()] + return {"ok": True, "value": value, "choices": choices} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) # type: ignore + + @app.get("/personalities/save_raw") + async def _save_raw_get(name: str, instructions: str = "", tools_text: str = "", voice: str = "marin") -> dict: # type: ignore + name_s = _sanitize_name(name) + if not name_s: + return JSONResponse({"ok": False, "error": "invalid_name"}, status_code=400) # type: ignore + try: + logger.info( + "Headless save_raw(GET): name=%r voice=%r instr_len=%d tools_len=%d", + name_s, + voice, + len(instructions), + len(tools_text), + ) + _write_profile(name_s, instructions, tools_text, voice or "marin") + value = f"user_personalities/{name_s}" + choices = [DEFAULT_OPTION, *list_personalities()] + return {"ok": True, "value": value, "choices": choices} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) # type: ignore + + logger = logging.getLogger(__name__) + + @app.post("/personalities/apply") + async def _apply( + payload: ApplyPayload | None = None, + name: str | None = None, + persist: Optional[bool] = None, + request: Optional[Request] = None, + ) -> dict: # type: ignore + if LOCKED_PROFILE is not None: + return JSONResponse( + {"ok": False, "error": "profile_locked", "locked_to": LOCKED_PROFILE}, + status_code=403, + ) # type: ignore + loop = get_loop() + if loop is None: + return JSONResponse({"ok": False, "error": "loop_unavailable"}, status_code=503) # type: ignore + + # Accept both JSON payload and query param for convenience + sel_name: Optional[str] = None + persist_flag = bool(persist) if persist is not None else False + if payload and getattr(payload, "name", None): + sel_name = payload.name + persist_flag = bool(getattr(payload, "persist", False)) + elif name: + sel_name = name + elif request is not None: + try: + body = await request.json() + if isinstance(body, dict) and body.get("name"): + sel_name = str(body.get("name")) + if isinstance(body, dict) and "persist" in body: + persist_flag = bool(body.get("persist")) + except Exception: + sel_name = None + if request is not None: + try: + q_persist = request.query_params.get("persist") + if q_persist is not None: + persist_flag = str(q_persist).lower() in {"1", "true", "yes", "on"} + except Exception: + pass + if not sel_name: + sel_name = DEFAULT_OPTION + + async def _do_apply() -> str: + sel = None if sel_name == DEFAULT_OPTION else sel_name + status = await handler.apply_personality(sel) + return status + + try: + logger.info("Headless apply: requested name=%r", sel_name) + fut = asyncio.run_coroutine_threadsafe(_do_apply(), loop) + status = fut.result(timeout=10) + persisted_choice = _startup_choice() + if persist_flag and persist_personality is not None: + try: + persist_personality(None if sel_name == DEFAULT_OPTION else sel_name) + persisted_choice = _startup_choice() + except Exception as e: + logger.warning("Failed to persist startup personality: %s", e) + return {"ok": True, "status": status, "startup": persisted_choice} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) # type: ignore + + @app.get("/voices") + async def _voices() -> list[str]: + loop = get_loop() + if loop is None: + return ["marin"] + + async def _get_v() -> list[str]: + try: + return await handler.get_available_voices() + except Exception: + return ["marin"] + + try: + fut = asyncio.run_coroutine_threadsafe(_get_v(), loop) + return fut.result(timeout=10) + except Exception: + return ["marin"] diff --git a/src/reachy_mini_receptionist/ical_calendar.py b/src/reachy_mini_receptionist/ical_calendar.py new file mode 100644 index 0000000000000000000000000000000000000000..f8ee88d5ff97f4074dbd977adbbfc850068a1bfe --- /dev/null +++ b/src/reachy_mini_receptionist/ical_calendar.py @@ -0,0 +1,248 @@ +"""iCal calendar source for the receptionist. + +When ``RECEPTION_ICS_URL`` is set in the environment, ``calendar_data`` +uses this module to fetch today's appointments. The URL is typically a +Google Calendar "Public address in iCal format" link +(Settings -> Integrate calendar -> Public address in iCal format). +When the URL is unset, ``calendar_data`` returns an empty schedule and +the receptionist serves walk-in visitors only (via ``lookup_employee``). + +Operator convention for event titles: + + " with " + +Examples: + + "Rohan Verma with Mukul" + "Sara Khan with Priya" + "David Lee with Arjun Mehta" + +The host name (or alias) is matched against ``employees.py``. Add an +optional " — note" suffix to the title or use the event's DESCRIPTION +field for the note. The event's LOCATION field is used as a fallback host +when the title doesn't contain " with ". + +Single-occurrence events only — RRULE recurrence is not expanded in v1. +""" +from __future__ import annotations + +import logging +import os +import time +from datetime import date, datetime +from typing import Any, Dict, List, Optional, Tuple +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +import httpx +from icalendar import Calendar + +logger = logging.getLogger(__name__) + +_CACHE_TTL_SECONDS: float = 300.0 +_HTTP_TIMEOUT_SECONDS: float = 10.0 + +_cache: Dict[str, Any] = {"fetched_at": 0.0, "data": [], "url": None, "valid": False} + + +def _display_tz() -> Any: + """Return the timezone to use for displaying iCal event times. + + Read from ``RECEPTION_TIMEZONE`` env var (e.g. ``Asia/Kolkata``, + ``America/New_York``). Defaults to ``Asia/Kolkata`` since the pilot + deployment is in India. Falls back to system local time on a bad + value rather than crashing the whole calendar fetch. + """ + raw = (os.getenv("RECEPTION_TIMEZONE") or "Asia/Kolkata").strip() + try: + return ZoneInfo(raw) + except ZoneInfoNotFoundError: + logger.warning( + "RECEPTION_TIMEZONE=%r is not a valid IANA tz name — " + "falling back to system local time", + raw, + ) + return None # signals "use astimezone() default" + + +# Visitor/host separators the operator may write in event titles. Order +# matters — we try the most-specific phrasing first so "is here to see" +# wins over a bare "to" if both appear. Case-insensitive match. +_VISITOR_HOST_SEPARATORS: Tuple[str, ...] = ( + " is here to see ", + " here to see ", + " to see ", + " meets with ", + " meeting with ", + " meeting ", + " meets ", + # Bare "meet" — added 2026-05-21 after the operator's calendar used + # the imperative form ("Krishna Meet Rohan", "Alex Meet Arjun"). Without + # this the title fails to split, host_query stays empty, and the bot + # falls back to a context-derived email instead of looking the host + # up in the employee directory. + " meet ", + " with ", + " for ", + " -> ", + " → ", +) + + +def _parse_summary(summary: str) -> Tuple[str, str, Optional[str]]: + """Split a SUMMARY into (visitor_name, host_query, inline_note). + + Visitor/host separator: any of ``_VISITOR_HOST_SEPARATORS`` + (case-insensitive). Note separator (applied to the rest after host + extraction): ``" — "``, ``" - "``, ``": "``. + + Returns ``(text, "", None)`` if no visitor/host separator is found — + the caller can then fall back to LOCATION for the host. + """ + if not summary: + return ("", "", None) + text = summary.strip() + lower = text.lower() + sep_idx = -1 + sep_len = 0 + for sep in _VISITOR_HOST_SEPARATORS: + idx = lower.find(sep) + if idx >= 0 and (sep_idx < 0 or idx < sep_idx): + sep_idx = idx + sep_len = len(sep) + if sep_idx < 0: + return (text, "", None) + visitor = text[:sep_idx].strip() + rest = text[sep_idx + sep_len:].strip() + note: Optional[str] = None + for delim in (" — ", " - ", ": "): + d = rest.find(delim) + if d >= 0: + note = rest[d + len(delim):].strip() + rest = rest[:d].strip() + break + return (visitor, rest, note) + + +def _format_time(dt: Any) -> str: + """Format a datetime as 'H:MM AM/PM' in the display timezone. + + iCal feeds frequently serialise event times in UTC (or with a TZID + pointing to a different region). We convert to ``RECEPTION_TIMEZONE`` + (default ``Asia/Kolkata``) before formatting so an operator + scheduling "1 PM" in IST sees "1:00 PM" on the dashboard, not + "7:30 AM" (UTC) — regardless of what timezone the robot's OS is in. + """ + if isinstance(dt, datetime): + if dt.tzinfo is not None: + tz = _display_tz() + local = dt.astimezone(tz) if tz is not None else dt.astimezone() + else: + local = dt + return local.strftime("%I:%M %p").lstrip("0") + return "all day" + + +def _local_event_date(start: Any) -> date: + """Return the display-tz date of an event's DTSTART.""" + if isinstance(start, datetime): + if start.tzinfo is not None: + tz = _display_tz() + return (start.astimezone(tz) if tz is not None else start.astimezone()).date() + return start.date() + return start # already a date + + +def _today_events(cal: Calendar, today: date) -> List[Dict[str, Any]]: + """Walk a parsed Calendar and return the events that fall on ``today``.""" + out: List[Dict[str, Any]] = [] + for event in cal.walk("VEVENT"): + dtstart = event.get("DTSTART") + if dtstart is None: + continue + start = dtstart.dt + if _local_event_date(start) != today: + continue + summary = str(event.get("SUMMARY") or "") + description = str(event.get("DESCRIPTION") or "").strip() + location = str(event.get("LOCATION") or "").strip() + visitor, host_from_title, inline_note = _parse_summary(summary) + host_query = host_from_title or location + # Only surface a note when there's something beyond the title — never + # echo the SUMMARY back into ``note`` (that just makes the LLM + # context noisier without adding information). + note = inline_note or description or "" + out.append({ + "time": _format_time(start), + "name": visitor or summary, + "note": note, + "_host_query": host_query, + "_dt": start, + }) + out.sort(key=_sort_key) + return out + + +def _sort_key(event: Dict[str, Any]) -> datetime: + dt = event.get("_dt") + if isinstance(dt, datetime): + return dt.replace(tzinfo=None) if dt.tzinfo is None else dt.astimezone().replace(tzinfo=None) + if isinstance(dt, date): + return datetime.combine(dt, datetime.min.time()) + return datetime.min + + +def _fetch_ics(url: str) -> str: + resp = httpx.get(url, timeout=_HTTP_TIMEOUT_SECONDS, follow_redirects=True) + resp.raise_for_status() + return resp.text + + +def fetch_appointments(ics_url: str, today: Optional[date] = None) -> List[Dict[str, Any]]: + """Return today's appointments from the iCal URL, cached for ~5 minutes. + + On any fetch/parse failure returns the last successful cache (or empty + list if there's no cache yet) and logs a warning. Each appointment dict + has keys: + + time (str) — "H:MM AM/PM" or "all day" + name (str) — visitor name parsed from SUMMARY + note (str) — inline note, DESCRIPTION, or SUMMARY fallback + _host_query (str) — host name/alias from SUMMARY ' with ' or LOCATION + _dt (datetime|date) — event start, for downstream use + + Resolution of ``_host_query`` to an email is the caller's job + (calendar_data.py uses ``employees.find_email_for``). + """ + if today is None: + tz = _display_tz() + today = (datetime.now(tz) if tz is not None else datetime.now().astimezone()).date() + now = time.time() + # Cache freshness is tracked by ``valid`` + ``fetched_at`` only. An empty + # list is a legitimate cached result ("no appointments today") — treating + # ``data`` truthiness as the freshness flag would force a re-fetch on + # every call on an empty-calendar day, which on the receptionist hot path + # blocks the audio loop with a synchronous HTTP call per request. + if ( + _cache["url"] == ics_url + and _cache["valid"] + and (now - _cache["fetched_at"]) < _CACHE_TTL_SECONDS + ): + return list(_cache["data"]) + try: + text = _fetch_ics(ics_url) + cal = Calendar.from_ical(text) + events = _today_events(cal, today) + _cache.update({"fetched_at": now, "data": events, "url": ics_url, "valid": True}) + logger.info("Fetched iCal: %d event(s) for %s", len(events), today) + return list(events) + except Exception as e: + logger.warning( + "iCal fetch failed (%s: %s); using last-good cache (%d entries)", + type(e).__name__, e, len(_cache.get("data", [])), + ) + return list(_cache.get("data", [])) + + +def clear_cache() -> None: + """Clear the iCal cache (test hook + manual refresh helper).""" + _cache.update({"fetched_at": 0.0, "data": [], "url": None, "valid": False}) diff --git a/src/reachy_mini_receptionist/images/reachymini_avatar.png b/src/reachy_mini_receptionist/images/reachymini_avatar.png new file mode 100644 index 0000000000000000000000000000000000000000..cb6dee522f40a200b6ddadc13b2afecefbf14f78 --- /dev/null +++ b/src/reachy_mini_receptionist/images/reachymini_avatar.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a63ac8802ff3542f01292c431c5278296880d74cd3580d219fcf4827bc235f9 +size 1228716 diff --git a/src/reachy_mini_receptionist/images/user_avatar.png b/src/reachy_mini_receptionist/images/user_avatar.png new file mode 100644 index 0000000000000000000000000000000000000000..c297bda9088aa6961fe49c3176ec20a0886149be --- /dev/null +++ b/src/reachy_mini_receptionist/images/user_avatar.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97ca125a86bacdaa41c8dca88abd9ca746fd5c9391eda24249c012432b0219b +size 1111878 diff --git a/src/reachy_mini_receptionist/main.py b/src/reachy_mini_receptionist/main.py new file mode 100644 index 0000000000000000000000000000000000000000..6dca3d13ebcb5d89ff824044aed923ba9a3f94b0 --- /dev/null +++ b/src/reachy_mini_receptionist/main.py @@ -0,0 +1,1199 @@ +"""Entrypoint for the Reachy Mini Receptionist app. + +Changes from the base realtime app template: +- FaceDatabase and FaceRecognitionWorker are initialised here and injected into + ToolDependencies (face_worker + face_db fields). +- A /video_feed MJPEG endpoint and /api/* JSON endpoints are mounted on the + FastAPI settings_app so the dashboard can show the live annotated camera feed, + guest list, calendar, and notifications. +- The camera_worker slot is left intact for future head-tracking integration. +""" + +import os +import sys +import time +import asyncio +import argparse +import threading +from pathlib import Path +from typing import Any, Dict, List, Optional + +import gradio as gr +from fastapi import FastAPI +from fastapi.responses import StreamingResponse, JSONResponse +from fastapi.staticfiles import StaticFiles +from fastrtc import Stream +from gradio.utils import get_space + +from reachy_mini import ReachyMini, ReachyMiniApp +from reachy_mini_receptionist.utils import ( + parse_args, + setup_logger, + handle_vision_stuff, + log_connection_troubleshooting, +) + + +def _mount_dashboard_api( + app: Any, + face_worker: Any, + face_db: Any, + realtime_handler: Any | None = None, + session_manager: Any | None = None, + visitor_log: Any | None = None, + employee_store: Any | None = None, + instance_path: Optional[str] = None, +) -> None: + """Mount receptionist dashboard API endpoints on a FastAPI app. + + Endpoints: + GET /dashboard → serves the dashboard HTML page + GET /video_feed → MJPEG stream of annotated camera frames + GET /api/guests → JSON list of registered guests + GET /api/calendar → JSON today's appointments + GET /api/outbox → JSON email outbox log + GET /api/face_status → JSON current face detection state + GET /api/face_event_last_sent → JSON last external face event sent to model + GET /api/logs → JSON recent face recognition debug logs + GET /api/session → JSON current visitor session state (state machine) + GET /api/session_event_last_sent → JSON last session context event sent to model + """ + if app is None or not hasattr(app, "get"): + return # No FastAPI app available (e.g. LocalStream without settings_app) + + # Body needs to be in scope for every POST/PATCH endpoint with a JSON + # body below. Imported once here at the top of the function so the + # order endpoints are defined in doesn't matter. + from fastapi import Body # noqa: F401 — used by endpoints below + + from reachy_mini_receptionist.calendar_data import get_appointments + from reachy_mini_receptionist.tools.send_email import get_outbox + + static_dir = Path(__file__).parent / "static" + dashboard_html = static_dir / "dashboard.html" + + # Serve guest face thumbnails as static files + guests_dir = face_db.guests_dir + app.mount( + "/guest_images", + StaticFiles(directory=str(guests_dir)), + name="guest_images", + ) + + # Serve dashboard page + if dashboard_html.exists(): + from fastapi.responses import FileResponse + + @app.get("/dashboard") + def _dashboard_page(): + return FileResponse(str(dashboard_html)) + + # MJPEG video feed + def _mjpeg_generator(): + boundary = b"--frame" + while True: + jpeg = face_worker.latest_annotated_jpeg + if jpeg: + yield ( + boundary + b"\r\nContent-Type: image/jpeg\r\n\r\n" + + jpeg + b"\r\n" + ) + time.sleep(0.05) # ~20 fps max + + @app.get("/video_feed") + def _video_feed(): + return StreamingResponse( + _mjpeg_generator(), + media_type="multipart/x-mixed-replace; boundary=frame", + ) + + @app.get("/api/guests") + def _api_guests(): + return JSONResponse(face_db.get_all_guests()) + + @app.delete("/api/guests") + def _api_delete_guest(name: str): + removed = face_db.delete_guest(name) + if not removed: + return JSONResponse( + {"ok": False, "error": "Guest not found", "name": name}, + status_code=404, + ) + return JSONResponse({"ok": True, "name": name}) + + @app.get("/api/calendar") + def _api_calendar(): + return JSONResponse(get_appointments()) + + @app.get("/api/outbox") + def _api_outbox(): + return JSONResponse(get_outbox()) + + @app.get("/api/face_status") + def _api_face_status(): + return JSONResponse({ + "name": face_worker.current_name, + "confidence": round(face_worker.confidence, 2), + "is_known": face_worker.current_name not in ("Unknown", "No face"), + }) + + @app.get("/api/face_event_last_sent") + def _api_face_event_last_sent(): + if realtime_handler is None or not hasattr(realtime_handler, "get_last_face_event_sent"): + return JSONResponse({"sent": False}) + + event = realtime_handler.get_last_face_event_sent() + if event is None: + return JSONResponse({"sent": False}) + + return JSONResponse({"sent": True, **event}) + + @app.get("/api/logs") + def _api_logs(): + return JSONResponse({"logs": face_worker.get_recent_logs(100)}) + + @app.get("/api/config") + def _api_config(): + from reachy_mini_receptionist.config import config + return JSONResponse({"model": config.MODEL_NAME}) + + @app.get("/api/best_face_jpeg") + def _api_best_face_jpeg(): + # Return the best face crop from the last 5 seconds as JPEG. + # When no face is available we return 204 (No Content) instead of + # 404 so the dashboard doesn't pollute the browser's network tab + # with red error rows every 500 ms while the room is empty. + import cv2 + from fastapi.responses import Response + + # Dashboard preview should show best currently available face immediately + # and not wait for dwell-based stabilization. + result = face_worker.best_recent_face(window_seconds=5.0, require_dwell=False) + name, conf, crop = result + + if crop is None: + return Response(status_code=204) + + # Encode 100x100 grayscale crop as JPEG (upsample to 200x200 for readability) + display = cv2.resize(crop, (200, 200), interpolation=cv2.INTER_NEAREST) + _, jpeg_buf = cv2.imencode(".jpg", display, [cv2.IMWRITE_JPEG_QUALITY, 85]) + jpeg_bytes = jpeg_buf.tobytes() + + # Find face area of the best entry for the header + face_area = 0 + with face_worker._lock: + if face_worker._detection_window: + best_entry = max(face_worker._detection_window, key=lambda e: e[1]) + face_area = best_entry[1] + + return Response( + content=jpeg_bytes, + media_type="image/jpeg", + headers={ + "X-Face-Name": name, + "X-Face-Confidence": str(round(conf, 2)), + "X-Face-Area": str(face_area), + "Cache-Control": "no-cache, no-store", + }, + ) + + @app.get("/api/session") + def _api_session(): + if session_manager is None: + return JSONResponse({"available": False}) + snap = session_manager.session + payload = snap.to_dict() if hasattr(snap, "to_dict") else {} + return JSONResponse({"available": True, **payload}) + + @app.get("/api/session_event_last_sent") + def _api_session_event_last_sent(): + if realtime_handler is None or not hasattr(realtime_handler, "get_last_session_event_sent"): + return JSONResponse({"sent": False}) + event = realtime_handler.get_last_session_event_sent() + if event is None: + return JSONResponse({"sent": False}) + return JSONResponse({"sent": True, **event}) + + @app.get("/api/visitor_log") + def _api_visitor_log(limit: int = 100): + if visitor_log is None: + return JSONResponse({"available": False, "today_count": 0, "visits": []}) + return JSONResponse({ + "available": True, + "today_count": visitor_log.count_today(), + "visits": visitor_log.list_visits(limit=limit), + }) + + @app.get("/api/visitor_log.csv") + def _api_visitor_log_csv(): + """Download the full visitor log as CSV (for HR / facilities handoff).""" + from fastapi.responses import Response + import csv + import io + + if visitor_log is None: + return Response(content="", media_type="text/csv") + + rows = visitor_log.list_visits(limit=1000) + buf = io.StringIO() + fieldnames = [ + "id", "started_at", "ended_at", + "visitor_name", "recognized_face_name", "employee_name", + "matched_appointment_time", "matched_appointment_note", + "email_sent_to", "final_state", "error_message", + ] + writer = csv.DictWriter(buf, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + for r in rows: + writer.writerow(r) + filename = f"visitor_log_{time.strftime('%Y-%m-%d')}.csv" + return Response( + content=buf.getvalue(), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + @app.delete("/api/visitor_log") + def _api_visitor_log_wipe(): + """Wipe every row of the visitor log. Scoped destructive action + used by the dashboard's panel-level Clear button. Does not touch + employees, settings, face DB, or session state.""" + if visitor_log is None: + return JSONResponse({"ok": False, "error": "visitor_log unavailable"}, status_code=503) + try: + removed = visitor_log.wipe_all() + return JSONResponse({"ok": True, "removed": removed}) + except Exception as e: + return JSONResponse( + {"ok": False, "error": f"{type(e).__name__}: {e}"}, status_code=500, + ) + + @app.get("/api/stats") + def _api_stats(): + """Consolidated stats strip data — visits, emails, last visit, state.""" + visits_today = visitor_log.count_today() if visitor_log is not None else 0 + emails_today = visitor_log.count_emails_delivered_today() if visitor_log is not None else 0 + last = visitor_log.last_visit() if visitor_log is not None else None + current_state = None + current_visitor = None + if session_manager is not None: + snap = session_manager.session + current_state = snap.current_state.value + current_visitor = snap.visitor_name + return JSONResponse({ + "visits_today": visits_today, + "emails_delivered_today": emails_today, + "last_visit": last, + "current_state": current_state, + "current_visitor": current_visitor, + }) + + @app.post("/api/session/reset") + def _api_session_reset(): + """Manual override: drop the current visitor session back to IDLE. + + Useful when the bot gets stuck (e.g. spurious face match keeps the + state from going back to idle). Mirrors the auto-timeout reset. + """ + if session_manager is None: + return JSONResponse({"ok": False, "error": "Session manager not available"}, status_code=503) + snap = session_manager.reset() + return JSONResponse({"ok": True, "current_state": snap.current_state.value}) + + @app.post("/api/guests/manual_register") + def _api_guests_manual_register(payload: dict = Body(...)): + """Operator-initiated visitor registration that bypasses voice. + + When the bot can't transcribe a visitor's name (common for short + non-English names that Whisper/gpt-realtime mangle), the operator + types it on the dashboard. The current camera face crop is saved + under that name AND the active session is flipped to RECOGNIZED so + downstream tools (lookup_employee / send_email) can proceed as if + register_guest had succeeded via voice. + """ + name = (payload.get("name") or "").strip() + if not name: + return JSONResponse( + {"ok": False, "error": "name is required"}, status_code=400, + ) + + worker = face_worker + if worker is None: + return JSONResponse( + {"ok": False, "error": "Face worker not available"}, + status_code=503, + ) + face_crop = getattr(worker, "current_encoding", None) + if face_crop is None: + return JSONResponse( + { + "ok": False, + "error": ( + "No face currently detected by the camera. Ask the " + "visitor to look directly at the lens and try again." + ), + }, + status_code=409, + ) + if face_db is None: + return JSONResponse( + {"ok": False, "error": "Face DB not available"}, + status_code=503, + ) + + try: + face_db.add_or_update_guest(name, face_crop) + try: + worker.rebuild_recognizer() + except Exception: + pass + except Exception as e: + return JSONResponse( + {"ok": False, "error": f"{type(e).__name__}: {e}"}, + status_code=500, + ) + + # Flip the active session so the LLM sees this as a confirmed + # registration and can resume the flow (calendar match / lookup / + # send_email) without re-asking for the name. + if session_manager is not None: + try: + from reachy_mini_receptionist.receptionist_state import ( + ReceptionState as _RS, + ) + session_manager.transition( + _RS.RECOGNIZED, + visitor_name=name, + recognized_face_name=name, + ) + except Exception as e: + print(f"[manual_register] session transition failed: {e}") + return JSONResponse({ + "ok": True, + "name": name, + "total_guests": face_db.count(), + }) + + @app.post("/api/demo/reset") + def _api_demo_reset(): + """Wipe everything that accumulates during testing so the next + demo runs from a clean slate. **Preserves** the employee + directory, calendar, and .env settings — operators don't want + to re-enter Mukul/Priya/etc. before every demo. + + Wipes: + - face DB (all guests/*.png) + - visitor log (every visit row) + - email outbox (in-memory) + - active session (forces IDLE) + + Triggers a face-recognizer rebuild so the worker sees the empty + DB on its next pass. + """ + from reachy_mini_receptionist.tools.send_email import clear_outbox + results = { + "guests_removed": 0, + "visits_removed": 0, + "outbox_removed": 0, + "session_reset": False, + "errors": [], + } + # Face DB + try: + if face_db is not None: + before = face_db.count() + face_db.clear() + results["guests_removed"] = before + except Exception as e: + results["errors"].append(f"face_db: {type(e).__name__}: {e}") + # Rebuild recognizer so the worker drops the wiped faces immediately + try: + if face_worker is not None and hasattr(face_worker, "rebuild_recognizer"): + face_worker.rebuild_recognizer() + except Exception as e: + results["errors"].append(f"face_worker: {type(e).__name__}: {e}") + # Visitor log + try: + if visitor_log is not None: + results["visits_removed"] = visitor_log.wipe_all() + except Exception as e: + results["errors"].append(f"visitor_log: {type(e).__name__}: {e}") + # Outbox + try: + results["outbox_removed"] = clear_outbox() + except Exception as e: + results["errors"].append(f"outbox: {type(e).__name__}: {e}") + # Session + try: + if session_manager is not None: + session_manager.reset() + results["session_reset"] = True + except Exception as e: + results["errors"].append(f"session: {type(e).__name__}: {e}") + results["ok"] = not results["errors"] + return JSONResponse(results) + + # ------------------------------------------------------------------ + # Employee CRUD — backs the Employees panel on the dashboard. + # (Body is imported at the top of _mount_dashboard_api.) + # ------------------------------------------------------------------ + + def _employee_store_or_503(): + if employee_store is None: + return JSONResponse( + {"ok": False, "error": "Employee store not available"}, status_code=503, + ) + return None + + @app.get("/api/employees") + def _api_employees_list(): + guard = _employee_store_or_503() + if guard is not None: + return guard + return JSONResponse({"employees": employee_store.list_all()}) + + @app.post("/api/employees") + def _api_employees_create(payload: dict = Body(...)): + guard = _employee_store_or_503() + if guard is not None: + return guard + try: + from reachy_mini_receptionist.employees_store import EmployeeExistsError + emp = employee_store.create( + name=payload.get("name", ""), + email=payload.get("email", ""), + aliases=payload.get("aliases") or [], + title=payload.get("title"), + ) + return JSONResponse({"ok": True, "employee": emp}) + except EmployeeExistsError as e: + return JSONResponse( + {"ok": False, "error": str(e)}, status_code=409, + ) + except ValueError as e: + return JSONResponse( + {"ok": False, "error": str(e)}, status_code=400, + ) + except Exception as e: + return JSONResponse( + {"ok": False, "error": f"{type(e).__name__}: {e}"}, status_code=500, + ) + + @app.patch("/api/employees/{employee_id}") + def _api_employees_update(employee_id: int, payload: dict = Body(...)): + guard = _employee_store_or_503() + if guard is not None: + return guard + try: + from reachy_mini_receptionist.employees_store import EmployeeExistsError + emp = employee_store.update( + employee_id, + name=payload.get("name"), + email=payload.get("email"), + aliases=payload.get("aliases"), + title=payload.get("title"), + ) + if emp is None: + return JSONResponse( + {"ok": False, "error": "Employee not found"}, status_code=404, + ) + return JSONResponse({"ok": True, "employee": emp}) + except EmployeeExistsError as e: + return JSONResponse( + {"ok": False, "error": str(e)}, status_code=409, + ) + except ValueError as e: + return JSONResponse( + {"ok": False, "error": str(e)}, status_code=400, + ) + except Exception as e: + return JSONResponse( + {"ok": False, "error": f"{type(e).__name__}: {e}"}, status_code=500, + ) + + @app.delete("/api/employees/{employee_id}") + def _api_employees_delete(employee_id: int): + guard = _employee_store_or_503() + if guard is not None: + return guard + removed = employee_store.delete(employee_id) + if not removed: + return JSONResponse( + {"ok": False, "error": "Employee not found"}, status_code=404, + ) + return JSONResponse({"ok": True}) + + # ------------------------------------------------------------------ + # Diagnostics — surfaces the kind of failures that turned today into + # a 30-min debugging session (audio at 7%, daemon asleep, OpenAI + # latency, etc) BEFORE the demo, not during. + # ------------------------------------------------------------------ + import socket + import subprocess + + def _check_tcp(host: str, port: int, timeout: float = 2.0) -> dict: + start = time.monotonic() + try: + with socket.create_connection((host, port), timeout=timeout): + return { + "ok": True, "latency_ms": int((time.monotonic() - start) * 1000), + } + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}"} + + def _check_daemon() -> dict: + try: + import httpx + t0 = time.monotonic() + r = httpx.get("http://localhost:8000/api/daemon/status", timeout=2.0) + latency = int((time.monotonic() - t0) * 1000) + if r.status_code != 200: + return {"ok": False, "latency_ms": latency, "error": f"HTTP {r.status_code}"} + body = r.json() + return { + "ok": body.get("state") == "started", + "latency_ms": latency, + "state": body.get("state"), + } + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}"} + + def _check_wifi() -> dict: + try: + out = subprocess.check_output( + ["iwconfig", "wlan0"], + stderr=subprocess.STDOUT, timeout=2.0, + ).decode("utf-8", errors="replace") + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}"} + info: dict = {"raw": out.strip()} + import re + m = re.search(r"Link Quality=(\d+)/(\d+)", out) + if m: + info["link_quality"] = int(m.group(1)) + info["link_quality_max"] = int(m.group(2)) + info["link_quality_pct"] = round(int(m.group(1)) / int(m.group(2)) * 100, 1) + m = re.search(r"Signal level=(-?\d+)\s*dBm", out) + if m: + info["signal_dbm"] = int(m.group(1)) + m = re.search(r"ESSID:\"([^\"]+)\"", out) + if m: + info["essid"] = m.group(1) + info["ok"] = info.get("link_quality_pct", 0) >= 50 if "link_quality_pct" in info else None + return info + + def _check_audio() -> dict: + try: + out = subprocess.check_output( + ["pactl", "list", "short", "sinks"], + stderr=subprocess.STDOUT, timeout=2.0, + ).decode("utf-8", errors="replace") + sinks = [line.split("\t")[1] if "\t" in line else line for line in out.splitlines() if line.strip()] + return {"ok": len(sinks) > 0, "sinks": sinks} + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}"} + + @app.get("/api/diagnostics/health") + def _api_diagnostics_health(): + from reachy_mini_receptionist.config import config as _cfg + results = { + "openai_realtime": _check_tcp("api.openai.com", 443), + "resend": _check_tcp("api.resend.com", 443), + "daemon": _check_daemon(), + "wifi": _check_wifi(), + "audio": _check_audio(), + "config": { + "openai_key_set": bool(_cfg.OPENAI_API_KEY), + "resend_key_set": bool(os.getenv("RESEND_API_KEY")), + "resend_from": os.getenv("RESEND_FROM", "onboarding@resend.dev"), + "ical_url_set": bool(os.getenv("RECEPTION_ICS_URL")), + "model": _cfg.MODEL_NAME, + }, + } + # Overall OK iff every "ok" we have is truthy (None counted as unknown -> ok) + overall = all( + (v.get("ok") in (True, None)) if isinstance(v, dict) else True + for k, v in results.items() if k != "config" + ) + return JSONResponse({"ok": overall, "checks": results}) + + # ------------------------------------------------------------------ + # Volume control — wraps pactl. Operators can change speaker volume + # without leaving the dashboard (previously required the Reachy Mini + # Control panel at :8000 or SSH'ing in). + # ------------------------------------------------------------------ + + # Reachy Mini uses ALSA directly (no PulseAudio), so we drive volume + # via `amixer`. The control name varies by device: Master is the + # standard ALSA name; PCM and Speaker are fallbacks on some Pi audio + # HATs; reachymini_audio_sink is the daemon's sink name on this image. + _AMIXER_CANDIDATES = ( + "Master", "PCM", "Speaker", "Headphone", "reachymini_audio_sink", + ) + + def _amixer_active_control() -> Optional[str]: + """Return the first amixer control that exists on this device.""" + try: + out = subprocess.check_output( + ["amixer", "scontrols"], + stderr=subprocess.STDOUT, timeout=2.0, + ).decode("utf-8", errors="replace") + except Exception: + return None + import re + names = re.findall(r"Simple mixer control '([^']+)'", out) + for cand in _AMIXER_CANDIDATES: + if cand in names: + return cand + return names[0] if names else None + + def _audio_get_volume() -> dict: + ctrl = _amixer_active_control() + if ctrl is None: + return {"ok": False, "error": "no amixer control found"} + try: + out = subprocess.check_output( + ["amixer", "sget", ctrl], + stderr=subprocess.STDOUT, timeout=2.0, + ).decode("utf-8", errors="replace") + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}"} + import re + m = re.search(r"\[(\d+)%\]", out) + percent = int(m.group(1)) if m else None + muted = "[off]" in out.lower() + return {"ok": True, "control": ctrl, "percent": percent, "muted": muted} + + @app.get("/api/audio/volume") + def _api_audio_volume_get(): + return JSONResponse(_audio_get_volume()) + + @app.post("/api/audio/volume") + def _api_audio_volume_set(payload: dict = Body(...)): + target = payload.get("percent") + if target is None: + return JSONResponse( + {"ok": False, "error": "percent (0-150) is required"}, + status_code=400, + ) + try: + pct = int(target) + except Exception: + return JSONResponse( + {"ok": False, "error": "percent must be an integer"}, + status_code=400, + ) + pct = max(0, min(150, pct)) + ctrl = _amixer_active_control() + if ctrl is None: + return JSONResponse( + {"ok": False, "error": "no amixer control found"}, + status_code=500, + ) + try: + subprocess.check_output( + ["amixer", "sset", ctrl, "unmute"], + stderr=subprocess.STDOUT, timeout=2.0, + ) + subprocess.check_output( + ["amixer", "sset", ctrl, f"{pct}%"], + stderr=subprocess.STDOUT, timeout=2.0, + ) + except Exception as e: + return JSONResponse( + {"ok": False, "error": f"{type(e).__name__}: {e}"}, + status_code=500, + ) + return JSONResponse({"ok": True, **_audio_get_volume()}) + + @app.post("/api/diagnostics/speaker_test") + def _api_diagnostics_speaker_test(): + try: + subprocess.Popen( + ["speaker-test", "-c", "1", "-t", "sine", "-f", "440", "-l", "1"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + return JSONResponse({"ok": True, "message": "Playing 1-second 440Hz tone"}) + except FileNotFoundError: + return JSONResponse( + {"ok": False, "error": "speaker-test binary not found"}, status_code=500, + ) + except Exception as e: + return JSONResponse( + {"ok": False, "error": f"{type(e).__name__}: {e}"}, status_code=500, + ) + + # ------------------------------------------------------------------ + # Settings — read/write a subset of .env keys via the dashboard so + # operators don't have to SSH and `nano .env` to set an API key. + # Sensitive values are masked on GET; full values written on PATCH. + # Changes that require app restart are flagged in the response. + # ------------------------------------------------------------------ + _SETTINGS_KEYS = { + "VOICE_BACKEND": {"secret": False, "restart": True}, + "GEMINI_LIVE_MODEL": {"secret": False, "restart": True}, + "GEMINI_LIVE_VOICE": {"secret": False, "restart": True}, + "OPENAI_API_KEY": {"secret": True, "restart": True}, + "GEMINI_API_KEY": {"secret": True, "restart": True}, + # GEMINI_MODEL removed 2026-05-21 — the name normalizer it + # configured is short-circuited in name_normalizer.py, so this + # key is unused. Re-add if/when the normalizer is reinstated. + "STT_MODEL": {"secret": False, "restart": True}, + "STT_DISABLE_BIAS": {"secret": False, "restart": True}, + "RESEND_API_KEY": {"secret": True, "restart": False}, + "RESEND_FROM": {"secret": False, "restart": False}, + "RECEPTION_ICS_URL": {"secret": False, "restart": False}, + "FACE_TTL_DAYS": {"secret": False, "restart": True}, + "VISITOR_LOG_RETENTION_DAYS": {"secret": False, "restart": True}, + "FACE_LBPH_THRESHOLD": {"secret": False, "restart": True}, + "MODEL_NAME": {"secret": False, "restart": True}, + } + + def _find_env_path() -> Optional[Path]: + try: + from dotenv import find_dotenv + p = find_dotenv(usecwd=True) + if p: + return Path(p) + except Exception: + pass + candidates = [] + if instance_path: + candidates.append(Path(instance_path) / ".env") + candidates.append(Path.cwd() / ".env") + for c in candidates: + if c.exists(): + return c + return candidates[0] if candidates else None + + def _mask(value: str) -> str: + if not value: + return "" + if len(value) <= 6: + return "•" * len(value) + return value[:3] + "•" * max(0, len(value) - 7) + value[-4:] + + @app.get("/api/settings") + def _api_settings_get(): + env_path = _find_env_path() + settings = [] + for key, meta in _SETTINGS_KEYS.items(): + raw = os.getenv(key) or "" + display = _mask(raw) if meta["secret"] and raw else raw + settings.append({ + "key": key, + "value": display, + "is_set": bool(raw), + "is_secret": meta["secret"], + "requires_restart": meta["restart"], + }) + return JSONResponse({ + "env_path": str(env_path) if env_path else None, + "settings": settings, + }) + + @app.patch("/api/settings") + def _api_settings_patch(payload: dict = Body(...)): + env_path = _find_env_path() + if env_path is None: + return JSONResponse( + {"ok": False, "error": "Could not locate .env file"}, status_code=500, + ) + env_path.parent.mkdir(parents=True, exist_ok=True) + # Read existing .env (if any), preserving comments + ordering. + lines: List[str] = [] + if env_path.exists(): + lines = env_path.read_text(encoding="utf-8").splitlines() + updates = { + k: str(v) for k, v in (payload or {}).items() + if k in _SETTINGS_KEYS and v is not None + } + if not updates: + return JSONResponse( + {"ok": False, "error": "No valid keys to update"}, status_code=400, + ) + # Rewrite — replace existing keys in place, append new ones. + seen: set[str] = set() + for i, line in enumerate(lines): + stripped = line.lstrip() + if not stripped or stripped.startswith("#"): + continue + if "=" not in stripped: + continue + key = stripped.split("=", 1)[0].strip() + if key in updates: + lines[i] = f"{key}={updates[key]}" + seen.add(key) + for key, val in updates.items(): + if key not in seen: + lines.append(f"{key}={val}") + env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + # Apply to current process where it's safe to do so without restart. + restart_required = False + for key, val in updates.items(): + os.environ[key] = val + if _SETTINGS_KEYS[key]["restart"]: + restart_required = True + return JSONResponse({ + "ok": True, + "updated": list(updates.keys()), + "restart_required": restart_required, + "env_path": str(env_path), + }) + + +def update_chatbot(chatbot: List[Dict[str, Any]], response: Dict[str, Any]) -> List[Dict[str, Any]]: + """Update the chatbot with AdditionalOutputs.""" + chatbot.append(response) + return chatbot + + +def main() -> None: + """Entrypoint for the Reachy Mini receptionist app.""" + args, _ = parse_args() + run(args) + + +def run( + args: argparse.Namespace, + robot: ReachyMini = None, + app_stop_event: Optional[threading.Event] = None, + settings_app: Optional[FastAPI] = None, + instance_path: Optional[str] = None, +) -> None: + """Run the Reachy Mini receptionist app.""" + # Importing runtime dependencies lazily keeps startup flexible across install contexts. + from reachy_mini_receptionist.moves import MovementManager + from reachy_mini_receptionist.console import LocalStream + from reachy_mini_receptionist.openai_realtime import OpenaiRealtimeHandler + from reachy_mini_receptionist.tools.core_tools import ToolDependencies + from reachy_mini_receptionist.audio.head_wobbler import HeadWobbler + + # Backend switch: VOICE_BACKEND=gemini (default) uses Google Gemini + # Live, "openai" uses OpenAI Realtime. Default flipped from openai to + # gemini on 2026-05-21 because (a) Gemini Live hears Indian names far + # more accurately than OpenAI's gpt-4o-transcribe, (b) operators on a + # free Gemini API key get more headroom than on free OpenAI credits. + # Imported lazily so a missing SDK doesn't break the alternate backend. + _voice_backend = (os.getenv("VOICE_BACKEND") or "gemini").strip().lower() + + logger = setup_logger(args.debug) + logger.info("Starting Reachy Mini Receptionist App") + + if args.no_camera and args.head_tracker is not None: + logger.warning( + "Head tracking disabled: --no-camera flag is set. " + "Remove --no-camera to enable head tracking." + ) + + if robot is None: + try: + robot_kwargs = {} + if args.robot_name is not None: + robot_kwargs["robot_name"] = args.robot_name + + logger.info("Initializing ReachyMini (SDK will auto-detect appropriate backend)") + robot = ReachyMini(**robot_kwargs) + + except TimeoutError as e: + logger.error( + "Connection timeout: Failed to connect to Reachy Mini daemon. " + f"Details: {e}" + ) + log_connection_troubleshooting(logger, args.robot_name) + sys.exit(1) + + except ConnectionError as e: + logger.error( + "Connection failed: Unable to establish connection to Reachy Mini. " + f"Details: {e}" + ) + log_connection_troubleshooting(logger, args.robot_name) + sys.exit(1) + + except Exception as e: + logger.error( + f"Unexpected error during robot initialization: {type(e).__name__}: {e}" + ) + logger.error("Please check your configuration and try again.") + sys.exit(1) + + # Auto-enable Gradio in simulation mode (both MuJoCo for daemon and mockup-sim for desktop app) + status = robot.client.get_status() + if isinstance(status, dict): + simulation_enabled = status.get("simulation_enabled", False) + mockup_sim_enabled = status.get("mockup_sim_enabled", False) + else: + simulation_enabled = getattr(status, "simulation_enabled", False) + mockup_sim_enabled = getattr(status, "mockup_sim_enabled", False) + + is_simulation = simulation_enabled or mockup_sim_enabled + + if is_simulation and not args.gradio: + logger.info("Simulation mode detected. Automatically enabling gradio flag.") + args.gradio = True + + camera_worker, _, vision_manager = handle_vision_stuff(args, robot) + + # ------------------------------------------------------------------ + # Receptionist: Face DB + Face Recognition Worker + # ------------------------------------------------------------------ + from reachy_mini_receptionist.face_db import FaceDatabase + from reachy_mini_receptionist.face_recognition_worker import FaceRecognitionWorker + + db_dir = Path(instance_path) if instance_path else Path.cwd() + face_db = FaceDatabase(db_dir / "guests.db") + face_worker = FaceRecognitionWorker(face_db, camera_worker=camera_worker) + + # ------------------------------------------------------------------ + # Receptionist: Session state machine + visitor log + # ------------------------------------------------------------------ + from reachy_mini_receptionist.session_manager import SessionManager + from reachy_mini_receptionist.conversation_controller import ConversationController + from reachy_mini_receptionist.visitor_log import VisitorLog + + visitor_log = VisitorLog(db_dir / "visitor_log.db") + session_manager = SessionManager(visitor_log=visitor_log) + conversation_controller = ConversationController(session_manager) + + # Employee directory — SQLite-backed CRUD. Seeded from the hardcoded + # _SEED_EMPLOYEES list in employees.py on a brand-new install; after + # that, the dashboard's Employees panel is the source of truth. + from reachy_mini_receptionist.employees_store import EmployeeStore + from reachy_mini_receptionist import employees as _employees_module + employee_store = EmployeeStore(db_dir / "employees.db") + try: + seeded = employee_store.seed_if_empty(_employees_module._SEED_EMPLOYEES) + if seeded: + print(f"[employees] Seeded {seeded} employee(s) on first run") + except Exception as e: + print(f"[employees] Seed failed: {e}") + _employees_module.set_store(employee_store) + + # Privacy retention — best-effort cleanup at startup. Defaults match the + # Day-2 plan (face TTL 30d, visit log 90d). Set the env var to 0 to + # disable either one. Runs once per app start — restart weekly or add a + # scheduled task if you keep the app up for months. + try: + face_ttl = float(os.getenv("FACE_TTL_DAYS", "30")) + removed_faces = face_db.cleanup_older_than(face_ttl) + if removed_faces: + print(f"[retention] Face DB: removed {removed_faces} guest(s) older than {face_ttl} days") + except Exception as e: + print(f"[retention] Face DB cleanup failed: {e}") + try: + log_retention = float(os.getenv("VISITOR_LOG_RETENTION_DAYS", "90")) + removed_visits = visitor_log.cleanup_older_than(log_retention) + if removed_visits: + print(f"[retention] Visitor log: removed {removed_visits} row(s) older than {log_retention} days") + except Exception as e: + print(f"[retention] Visitor log cleanup failed: {e}") + + movement_manager = MovementManager( + current_robot=robot, + camera_worker=camera_worker, + ) + + head_wobbler = HeadWobbler(set_speech_offsets=movement_manager.set_speech_offsets) + + deps = ToolDependencies( + reachy_mini=robot, + movement_manager=movement_manager, + camera_worker=camera_worker, + vision_manager=vision_manager, + head_wobbler=head_wobbler, + face_worker=face_worker, + face_db=face_db, + session_manager=session_manager, + conversation_controller=conversation_controller, + ) + current_file_path = os.path.dirname(os.path.abspath(__file__)) + logger.debug(f"Current file absolute path: {current_file_path}") + chatbot = gr.Chatbot( + type="messages", + resizable=True, + avatar_images=( + os.path.join(current_file_path, "images", "user_avatar.png"), + os.path.join(current_file_path, "images", "reachymini_avatar.png"), + ), + ) + logger.debug(f"Chatbot avatar images: {chatbot.avatar_images}") + + if _voice_backend == "gemini": + from reachy_mini_receptionist.gemini_live import GeminiLiveHandler + logger.info("VOICE_BACKEND=gemini — using Gemini Live handler") + handler = GeminiLiveHandler( + deps, + gradio_mode=args.gradio, + instance_path=instance_path, + session_manager=session_manager, + controller=conversation_controller, + ) + else: + logger.info("VOICE_BACKEND=openai (default) — using OpenAI Realtime handler") + handler = OpenaiRealtimeHandler( + deps, + gradio_mode=args.gradio, + instance_path=instance_path, + session_manager=session_manager, + controller=conversation_controller, + ) + + def _face_event_forwarder(event: Dict[str, Any]) -> None: + """Route a face event through the controller, then to the LLM context.""" + try: + conversation_controller.on_face_event(event) + except Exception as exc: + logger.warning("ConversationController.on_face_event raised %s: %s", type(exc).__name__, exc) + handler.notify_external_face_event(event) + + face_worker.set_face_event_callback(_face_event_forwarder) + + # Subscribe the handler BEFORE face_worker.start() so the first + # transitions aren't dropped. SessionManager buffers events that fire + # before the realtime websocket is connected. + session_manager.subscribe(handler.notify_session_event) + + stream_manager: gr.Blocks | LocalStream | None = None + + if args.gradio: + api_key_textbox = gr.Textbox( + label="OPENAI API Key", + type="password", + value=os.getenv("OPENAI_API_KEY") if not get_space() else "", + ) + + from reachy_mini_receptionist.gradio_personality import PersonalityUI + + personality_ui = PersonalityUI() + personality_ui.create_components() + + stream = Stream( + handler=handler, + mode="send-receive", + modality="audio", + additional_inputs=[ + chatbot, + api_key_textbox, + *personality_ui.additional_inputs_ordered(), + ], + additional_outputs=[chatbot], + additional_outputs_handler=update_chatbot, + ui_args={"title": "Talk with Reachy Mini"}, + ) + stream_manager = stream.ui + if not settings_app: + app = FastAPI() + else: + app = settings_app + + personality_ui.wire_events(handler, stream_manager) + + # ------------------------------------------------------------------ + # Mount dashboard API endpoints BEFORE wrapping with Gradio so that + # /video_feed, /api/*, /dashboard routes are available on the same app. + # ------------------------------------------------------------------ + _mount_dashboard_api( + app, face_worker, face_db, handler, session_manager, visitor_log, + employee_store=employee_store, instance_path=instance_path, + ) + logger.info("📊 Receptionist dashboard available at: http://localhost:7860/dashboard") + + app = gr.mount_gradio_app(app, stream.ui, path="/") + else: + # In headless mode, wire settings_app + instance_path to console LocalStream + stream_manager = LocalStream( + handler, + robot, + settings_app=settings_app, + instance_path=instance_path, + ) + + # ------------------------------------------------------------------ + # Mount dashboard API endpoints on settings_app when headless + # ------------------------------------------------------------------ + _mount_dashboard_api( + settings_app, face_worker, face_db, handler, session_manager, visitor_log, + employee_store=employee_store, instance_path=instance_path, + ) + + # Each async service → its own thread/loop + movement_manager.start() + head_wobbler.start() + face_worker.start() + if camera_worker: + camera_worker.start() + if vision_manager: + vision_manager.start() + + def poll_stop_event() -> None: + """Poll the stop event to allow graceful shutdown.""" + if app_stop_event is not None: + app_stop_event.wait() + + logger.info("App stop event detected, shutting down...") + try: + stream_manager.close() + except Exception as e: + logger.error(f"Error while closing stream manager: {e}") + + if app_stop_event: + threading.Thread(target=poll_stop_event, daemon=True).start() + + try: + stream_manager.launch() + except KeyboardInterrupt: + logger.info("Keyboard interruption in main thread... closing server.") + finally: + movement_manager.stop() + head_wobbler.stop() + face_worker.stop() + if camera_worker: + camera_worker.stop() + if vision_manager: + vision_manager.stop() + + # Ensure media is explicitly closed before disconnecting + try: + robot.media.close() + except Exception as e: + logger.debug(f"Error closing media during shutdown: {e}") + + # prevent connection to keep alive some threads + robot.client.disconnect() + time.sleep(1) + logger.info("Shutdown complete.") + + +class ReachyMiniReceptionist(ReachyMiniApp): # type: ignore[misc] + """Reachy Mini Apps entry point for the receptionist app.""" + + custom_app_url = "http://0.0.0.0:7860/" + dont_start_webserver = False + + def run(self, reachy_mini: ReachyMini, stop_event: threading.Event) -> None: + """Run the Reachy Mini receptionist app.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + args, _ = parse_args() + + # is_wireless = reachy_mini.client.get_status()["wireless_version"] + # args.head_tracker = None if is_wireless else "mediapipe" + + instance_path = self._get_instance_path().parent + run( + args, + robot=reachy_mini, + app_stop_event=stop_event, + settings_app=self.settings_app, + instance_path=instance_path, + ) + + +if __name__ == "__main__": + app = ReachyMiniReceptionist() + try: + app.wrapped_run() + except KeyboardInterrupt: + app.stop() diff --git a/src/reachy_mini_receptionist/moves.py b/src/reachy_mini_receptionist/moves.py new file mode 100644 index 0000000000000000000000000000000000000000..3c319e73d70519ea837987285849b997db2032a0 --- /dev/null +++ b/src/reachy_mini_receptionist/moves.py @@ -0,0 +1,849 @@ +"""Movement system with sequential primary moves and additive secondary moves. + +Design overview +- Primary moves (emotions, dances, goto, breathing) are mutually exclusive and run + sequentially. +- Secondary moves (speech sway, face tracking) are additive offsets applied on top + of the current primary pose. +- There is a single control point to the robot: `ReachyMini.set_target`. +- The control loop runs near 100 Hz and is phase-aligned via a monotonic clock. +- Idle behaviour starts an infinite `BreathingMove` after a short inactivity delay + unless listening is active. + +Threading model +- A dedicated worker thread owns all real-time state and issues `set_target` + commands. +- Other threads communicate via a command queue (enqueue moves, mark activity, + toggle listening). +- Secondary offset producers set pending values guarded by locks; the worker + snaps them atomically. + +Units and frames +- Secondary offsets are interpreted as metres for x/y/z and radians for + roll/pitch/yaw in the world frame (unless noted by `compose_world_offset`). +- Antennas and `body_yaw` are in radians. +- Head pose composition uses `compose_world_offset(primary_head, secondary_head)`; + the secondary offset must therefore be expressed in the world frame. + +Safety +- Listening freezes antennas, then blends them back on unfreeze. +- Interpolations and blends are used to avoid jumps at all times. +- `set_target` errors are rate-limited in logs. +""" + +from __future__ import annotations +import time +import logging +import threading +from queue import Empty, Queue +from typing import Any, Dict, Tuple +from collections import deque +from dataclasses import dataclass + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini import ReachyMini +from reachy_mini.utils import create_head_pose +from reachy_mini.motion.move import Move +from reachy_mini.utils.interpolation import ( + compose_world_offset, + linear_pose_interpolation, +) + + +logger = logging.getLogger(__name__) + +# Configuration constants +CONTROL_LOOP_FREQUENCY_HZ = 100.0 # Hz - Target frequency for the movement control loop + +# Type definitions +FullBodyPose = Tuple[NDArray[np.float32], Tuple[float, float], float] # (head_pose_4x4, antennas, body_yaw) + + +class BreathingMove(Move): # type: ignore + """Breathing move with interpolation to neutral and then continuous breathing patterns.""" + + def __init__( + self, + interpolation_start_pose: NDArray[np.float32], + interpolation_start_antennas: Tuple[float, float], + interpolation_duration: float = 1.0, + ): + """Initialize breathing move. + + Args: + interpolation_start_pose: 4x4 matrix of current head pose to interpolate from + interpolation_start_antennas: Current antenna positions to interpolate from + interpolation_duration: Duration of interpolation to neutral (seconds) + + """ + self.interpolation_start_pose = interpolation_start_pose + self.interpolation_start_antennas = np.array(interpolation_start_antennas) + self.interpolation_duration = interpolation_duration + + # Neutral positions for breathing base + self.neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + self.neutral_antennas = np.array([0.0, 0.0]) + + # Breathing parameters + self.breathing_z_amplitude = 0.005 # 5mm gentle breathing + self.breathing_frequency = 0.1 # Hz (6 breaths per minute) + self.antenna_sway_amplitude = np.deg2rad(15) # 15 degrees + self.antenna_frequency = 0.5 # Hz (faster antenna sway) + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return float("inf") # Continuous breathing (never ends naturally) + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate breathing move at time t.""" + if t < self.interpolation_duration: + # Phase 1: Interpolate to neutral base position + interpolation_t = t / self.interpolation_duration + + # Interpolate head pose + head_pose = linear_pose_interpolation( + self.interpolation_start_pose, self.neutral_head_pose, interpolation_t, + ) + + # Interpolate antennas + antennas_interp = ( + 1 - interpolation_t + ) * self.interpolation_start_antennas + interpolation_t * self.neutral_antennas + antennas = antennas_interp.astype(np.float64) + + else: + # Phase 2: Breathing patterns from neutral base + breathing_time = t - self.interpolation_duration + + # Gentle z-axis breathing + z_offset = self.breathing_z_amplitude * np.sin(2 * np.pi * self.breathing_frequency * breathing_time) + head_pose = create_head_pose(x=0, y=0, z=z_offset, roll=0, pitch=0, yaw=0, degrees=True, mm=False) + + # Antenna sway (opposite directions) + antenna_sway = self.antenna_sway_amplitude * np.sin(2 * np.pi * self.antenna_frequency * breathing_time) + antennas = np.array([antenna_sway, -antenna_sway], dtype=np.float64) + + # Return in official Move interface format: (head_pose, antennas_array, body_yaw) + return (head_pose, antennas, 0.0) + + +def combine_full_body(primary_pose: FullBodyPose, secondary_pose: FullBodyPose) -> FullBodyPose: + """Combine primary and secondary full body poses. + + Args: + primary_pose: (head_pose, antennas, body_yaw) - primary move + secondary_pose: (head_pose, antennas, body_yaw) - secondary offsets + + Returns: + Combined full body pose (head_pose, antennas, body_yaw) + + """ + primary_head, primary_antennas, primary_body_yaw = primary_pose + secondary_head, secondary_antennas, secondary_body_yaw = secondary_pose + + # Combine head poses using compose_world_offset; the secondary pose must be an + # offset expressed in the world frame (T_off_world) applied to the absolute + # primary transform (T_abs). + combined_head = compose_world_offset(primary_head, secondary_head, reorthonormalize=True) + + # Sum antennas and body_yaw + combined_antennas = ( + primary_antennas[0] + secondary_antennas[0], + primary_antennas[1] + secondary_antennas[1], + ) + combined_body_yaw = primary_body_yaw + secondary_body_yaw + + return (combined_head, combined_antennas, combined_body_yaw) + + +def clone_full_body_pose(pose: FullBodyPose) -> FullBodyPose: + """Create a deep copy of a full body pose tuple.""" + head, antennas, body_yaw = pose + return (head.copy(), (float(antennas[0]), float(antennas[1])), float(body_yaw)) + + +@dataclass +class MovementState: + """State tracking for the movement system.""" + + # Primary move state + current_move: Move | None = None + move_start_time: float | None = None + last_activity_time: float = 0.0 + + # Secondary move state (offsets) + speech_offsets: Tuple[float, float, float, float, float, float] = ( + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ) + face_tracking_offsets: Tuple[float, float, float, float, float, float] = ( + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ) + + # Status flags + last_primary_pose: FullBodyPose | None = None + + def update_activity(self) -> None: + """Update the last activity time.""" + self.last_activity_time = time.monotonic() + + +@dataclass +class LoopFrequencyStats: + """Track rolling loop frequency statistics.""" + + mean: float = 0.0 + m2: float = 0.0 + min_freq: float = float("inf") + count: int = 0 + last_freq: float = 0.0 + potential_freq: float = 0.0 + + def reset(self) -> None: + """Reset accumulators while keeping the last potential frequency.""" + self.mean = 0.0 + self.m2 = 0.0 + self.min_freq = float("inf") + self.count = 0 + + +class MovementManager: + """Coordinate sequential moves, additive offsets, and robot output at 100 Hz. + + Responsibilities: + - Own a real-time loop that samples the current primary move (if any), fuses + secondary offsets, and calls `set_target` exactly once per tick. + - Start an idle `BreathingMove` after `idle_inactivity_delay` when not + listening and no moves are queued. + - Expose thread-safe APIs so other threads can enqueue moves, mark activity, + or feed secondary offsets without touching internal state. + + Timing: + - All elapsed-time calculations rely on `time.monotonic()` through `self._now` + to avoid wall-clock jumps. + - The loop attempts 100 Hz + + Concurrency: + - External threads communicate via `_command_queue` messages. + - Secondary offsets are staged via dirty flags guarded by locks and consumed + atomically inside the worker loop. + """ + + def __init__( + self, + current_robot: ReachyMini, + camera_worker: "Any" = None, + ): + """Initialize movement manager.""" + self.current_robot = current_robot + self.camera_worker = camera_worker + + # Single timing source for durations + self._now = time.monotonic + + # Movement state + self.state = MovementState() + self.state.last_activity_time = self._now() + neutral_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + self.state.last_primary_pose = (neutral_pose, (0.0, 0.0), 0.0) + + # Move queue (primary moves) + self.move_queue: deque[Move] = deque() + + # Configuration + self.idle_inactivity_delay = 0.3 # seconds + self.target_frequency = CONTROL_LOOP_FREQUENCY_HZ + self.target_period = 1.0 / self.target_frequency + + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + self._is_listening = False + self._last_commanded_pose: FullBodyPose = clone_full_body_pose(self.state.last_primary_pose) + self._listening_antennas: Tuple[float, float] = self._last_commanded_pose[1] + self._antenna_unfreeze_blend = 1.0 + self._antenna_blend_duration = 0.4 # seconds to blend back after listening + self._last_listening_blend_time = self._now() + self._breathing_active = False # true when breathing move is running or queued + self._listening_debounce_s = 0.15 + self._last_listening_toggle_time = self._now() + self._last_set_target_err = 0.0 + self._set_target_err_interval = 1.0 # seconds between error logs + self._set_target_err_suppressed = 0 + + # Cross-thread signalling + self._command_queue: "Queue[Tuple[str, Any]]" = Queue() + self._speech_offsets_lock = threading.Lock() + self._pending_speech_offsets: Tuple[float, float, float, float, float, float] = ( + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ) + self._speech_offsets_dirty = False + + self._face_offsets_lock = threading.Lock() + self._pending_face_offsets: Tuple[float, float, float, float, float, float] = ( + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ) + self._face_offsets_dirty = False + + self._shared_state_lock = threading.Lock() + self._shared_last_activity_time = self.state.last_activity_time + self._shared_is_listening = self._is_listening + self._status_lock = threading.Lock() + self._freq_stats = LoopFrequencyStats() + self._freq_snapshot = LoopFrequencyStats() + + def queue_move(self, move: Move) -> None: + """Queue a primary move to run after the currently executing one. + + Thread-safe: the move is enqueued via the worker command queue so the + control loop remains the sole mutator of movement state. + """ + self._command_queue.put(("queue_move", move)) + + def clear_move_queue(self) -> None: + """Stop the active move and discard any queued primary moves. + + Thread-safe: executed by the worker thread via the command queue. + """ + self._command_queue.put(("clear_queue", None)) + + def set_speech_offsets(self, offsets: Tuple[float, float, float, float, float, float]) -> None: + """Update speech-induced secondary offsets (x, y, z, roll, pitch, yaw). + + Offsets are interpreted as metres for translation and radians for + rotation in the world frame. Thread-safe via a pending snapshot. + """ + with self._speech_offsets_lock: + self._pending_speech_offsets = offsets + self._speech_offsets_dirty = True + + def set_moving_state(self, duration: float) -> None: + """Mark the robot as actively moving for the provided duration. + + Legacy hook used by goto helpers to keep inactivity and breathing logic + aware of manual motions. Thread-safe via the command queue. + """ + self._command_queue.put(("set_moving_state", duration)) + + def is_idle(self) -> bool: + """Return True when the robot has been inactive longer than the idle delay.""" + with self._shared_state_lock: + last_activity = self._shared_last_activity_time + listening = self._shared_is_listening + + if listening: + return False + + return self._now() - last_activity >= self.idle_inactivity_delay + + def set_listening(self, listening: bool) -> None: + """Enable or disable listening mode without touching shared state directly. + + While listening: + - Antenna positions are frozen at the last commanded values. + - Blending is reset so that upon unfreezing the antennas return smoothly. + - Idle breathing is suppressed. + + Thread-safe: the change is posted to the worker command queue. + """ + with self._shared_state_lock: + if self._shared_is_listening == listening: + return + self._command_queue.put(("set_listening", listening)) + + def _poll_signals(self, current_time: float) -> None: + """Apply queued commands and pending offset updates.""" + self._apply_pending_offsets() + + while True: + try: + command, payload = self._command_queue.get_nowait() + except Empty: + break + self._handle_command(command, payload, current_time) + + def _apply_pending_offsets(self) -> None: + """Apply the most recent speech/face offset updates.""" + speech_offsets: Tuple[float, float, float, float, float, float] | None = None + with self._speech_offsets_lock: + if self._speech_offsets_dirty: + speech_offsets = self._pending_speech_offsets + self._speech_offsets_dirty = False + + if speech_offsets is not None: + self.state.speech_offsets = speech_offsets + self.state.update_activity() + + face_offsets: Tuple[float, float, float, float, float, float] | None = None + with self._face_offsets_lock: + if self._face_offsets_dirty: + face_offsets = self._pending_face_offsets + self._face_offsets_dirty = False + + if face_offsets is not None: + self.state.face_tracking_offsets = face_offsets + self.state.update_activity() + + def _handle_command(self, command: str, payload: Any, current_time: float) -> None: + """Handle a single cross-thread command.""" + if command == "queue_move": + if isinstance(payload, Move): + self.move_queue.append(payload) + self.state.update_activity() + duration = getattr(payload, "duration", None) + if duration is not None: + try: + duration_str = f"{float(duration):.2f}" + except (TypeError, ValueError): + duration_str = str(duration) + else: + duration_str = "?" + logger.debug( + "Queued move with duration %ss, queue size: %s", + duration_str, + len(self.move_queue), + ) + else: + logger.warning("Ignored queue_move command with invalid payload: %s", payload) + elif command == "clear_queue": + self.move_queue.clear() + self.state.current_move = None + self.state.move_start_time = None + self._breathing_active = False + logger.info("Cleared move queue and stopped current move") + elif command == "set_moving_state": + try: + duration = float(payload) + except (TypeError, ValueError): + logger.warning("Invalid moving state duration: %s", payload) + return + self.state.update_activity() + elif command == "mark_activity": + self.state.update_activity() + elif command == "set_listening": + desired_state = bool(payload) + now = self._now() + if now - self._last_listening_toggle_time < self._listening_debounce_s: + return + self._last_listening_toggle_time = now + + if self._is_listening == desired_state: + return + + self._is_listening = desired_state + self._last_listening_blend_time = now + if desired_state: + # Freeze: snapshot current commanded antennas and reset blend + self._listening_antennas = ( + float(self._last_commanded_pose[1][0]), + float(self._last_commanded_pose[1][1]), + ) + self._antenna_unfreeze_blend = 0.0 + else: + # Unfreeze: restart blending from frozen pose + self._antenna_unfreeze_blend = 0.0 + self.state.update_activity() + else: + logger.warning("Unknown command received by MovementManager: %s", command) + + def _publish_shared_state(self) -> None: + """Expose idle-related state for external threads.""" + with self._shared_state_lock: + self._shared_last_activity_time = self.state.last_activity_time + self._shared_is_listening = self._is_listening + + def _manage_move_queue(self, current_time: float) -> None: + """Manage the primary move queue (sequential execution).""" + if self.state.current_move is None or ( + self.state.move_start_time is not None + and current_time - self.state.move_start_time >= self.state.current_move.duration + ): + self.state.current_move = None + self.state.move_start_time = None + + if self.move_queue: + self.state.current_move = self.move_queue.popleft() + self.state.move_start_time = current_time + # Any real move cancels breathing mode flag + self._breathing_active = isinstance(self.state.current_move, BreathingMove) + logger.debug(f"Starting new move, duration: {self.state.current_move.duration}s") + + def _manage_breathing(self, current_time: float) -> None: + """Manage automatic breathing when idle.""" + if ( + self.state.current_move is None + and not self.move_queue + and not self._is_listening + and not self._breathing_active + ): + idle_for = current_time - self.state.last_activity_time + if idle_for >= self.idle_inactivity_delay: + try: + # These 2 functions return the latest available sensor data from the robot, but don't perform I/O synchronously. + # Therefore, we accept calling them inside the control loop. + _, current_antennas = self.current_robot.get_current_joint_positions() + current_head_pose = self.current_robot.get_current_head_pose() + + self._breathing_active = True + self.state.update_activity() + + breathing_move = BreathingMove( + interpolation_start_pose=current_head_pose, + interpolation_start_antennas=current_antennas, + interpolation_duration=1.0, + ) + self.move_queue.append(breathing_move) + logger.debug("Started breathing after %.1fs of inactivity", idle_for) + except Exception as e: + self._breathing_active = False + logger.error("Failed to start breathing: %s", e) + + if isinstance(self.state.current_move, BreathingMove) and self.move_queue: + self.state.current_move = None + self.state.move_start_time = None + self._breathing_active = False + logger.debug("Stopping breathing due to new move activity") + + if self.state.current_move is not None and not isinstance(self.state.current_move, BreathingMove): + self._breathing_active = False + + def _get_primary_pose(self, current_time: float) -> FullBodyPose: + """Get the primary full body pose from current move or neutral.""" + # When a primary move is playing, sample it and cache the resulting pose + if self.state.current_move is not None and self.state.move_start_time is not None: + move_time = current_time - self.state.move_start_time + head, antennas, body_yaw = self.state.current_move.evaluate(move_time) + + if head is None: + head = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + if antennas is None: + antennas = np.array([0.0, 0.0]) + if body_yaw is None: + body_yaw = 0.0 + + antennas_tuple = (float(antennas[0]), float(antennas[1])) + head_copy = head.copy() + primary_full_body_pose = ( + head_copy, + antennas_tuple, + float(body_yaw), + ) + + self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose) + # Otherwise reuse the last primary pose so we avoid jumps between moves + elif self.state.last_primary_pose is not None: + primary_full_body_pose = clone_full_body_pose(self.state.last_primary_pose) + else: + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + primary_full_body_pose = (neutral_head_pose, (0.0, 0.0), 0.0) + self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose) + + return primary_full_body_pose + + def _get_secondary_pose(self) -> FullBodyPose: + """Get the secondary full body pose from speech and face tracking offsets.""" + # Combine speech sway offsets + face tracking offsets for secondary pose + secondary_offsets = [ + self.state.speech_offsets[0] + self.state.face_tracking_offsets[0], + self.state.speech_offsets[1] + self.state.face_tracking_offsets[1], + self.state.speech_offsets[2] + self.state.face_tracking_offsets[2], + self.state.speech_offsets[3] + self.state.face_tracking_offsets[3], + self.state.speech_offsets[4] + self.state.face_tracking_offsets[4], + self.state.speech_offsets[5] + self.state.face_tracking_offsets[5], + ] + + secondary_head_pose = create_head_pose( + x=secondary_offsets[0], + y=secondary_offsets[1], + z=secondary_offsets[2], + roll=secondary_offsets[3], + pitch=secondary_offsets[4], + yaw=secondary_offsets[5], + degrees=False, + mm=False, + ) + return (secondary_head_pose, (0.0, 0.0), 0.0) + + def _compose_full_body_pose(self, current_time: float) -> FullBodyPose: + """Compose primary and secondary poses into a single command pose.""" + primary = self._get_primary_pose(current_time) + secondary = self._get_secondary_pose() + return combine_full_body(primary, secondary) + + def _update_primary_motion(self, current_time: float) -> None: + """Advance queue state and idle behaviours for this tick.""" + self._manage_move_queue(current_time) + self._manage_breathing(current_time) + + def _calculate_blended_antennas(self, target_antennas: Tuple[float, float]) -> Tuple[float, float]: + """Blend target antennas with listening freeze state and update blending.""" + now = self._now() + listening = self._is_listening + listening_antennas = self._listening_antennas + blend = self._antenna_unfreeze_blend + blend_duration = self._antenna_blend_duration + last_update = self._last_listening_blend_time + self._last_listening_blend_time = now + + if listening: + antennas_cmd = listening_antennas + new_blend = 0.0 + else: + dt = max(0.0, now - last_update) + if blend_duration <= 0: + new_blend = 1.0 + else: + new_blend = min(1.0, blend + dt / blend_duration) + antennas_cmd = ( + listening_antennas[0] * (1.0 - new_blend) + target_antennas[0] * new_blend, + listening_antennas[1] * (1.0 - new_blend) + target_antennas[1] * new_blend, + ) + + if listening: + self._antenna_unfreeze_blend = 0.0 + else: + self._antenna_unfreeze_blend = new_blend + if new_blend >= 1.0: + self._listening_antennas = ( + float(target_antennas[0]), + float(target_antennas[1]), + ) + + return antennas_cmd + + def _issue_control_command(self, head: NDArray[np.float32], antennas: Tuple[float, float], body_yaw: float) -> None: + """Send the fused pose to the robot with throttled error logging.""" + try: + self.current_robot.set_target(head=head, antennas=antennas, body_yaw=body_yaw) + except Exception as e: + now = self._now() + if now - self._last_set_target_err >= self._set_target_err_interval: + msg = f"Failed to set robot target: {e}" + if self._set_target_err_suppressed: + msg += f" (suppressed {self._set_target_err_suppressed} repeats)" + self._set_target_err_suppressed = 0 + logger.error(msg) + self._last_set_target_err = now + else: + self._set_target_err_suppressed += 1 + else: + with self._status_lock: + self._last_commanded_pose = clone_full_body_pose((head, antennas, body_yaw)) + + def _update_frequency_stats( + self, loop_start: float, prev_loop_start: float, stats: LoopFrequencyStats, + ) -> LoopFrequencyStats: + """Update frequency statistics based on the current loop start time.""" + period = loop_start - prev_loop_start + if period > 0: + stats.last_freq = 1.0 / period + stats.count += 1 + delta = stats.last_freq - stats.mean + stats.mean += delta / stats.count + stats.m2 += delta * (stats.last_freq - stats.mean) + stats.min_freq = min(stats.min_freq, stats.last_freq) + return stats + + def _schedule_next_tick(self, loop_start: float, stats: LoopFrequencyStats) -> Tuple[float, LoopFrequencyStats]: + """Compute sleep time to maintain target frequency and update potential freq.""" + computation_time = self._now() - loop_start + stats.potential_freq = 1.0 / computation_time if computation_time > 0 else float("inf") + sleep_time = max(0.0, self.target_period - computation_time) + return sleep_time, stats + + def _record_frequency_snapshot(self, stats: LoopFrequencyStats) -> None: + """Store a thread-safe snapshot of current frequency statistics.""" + with self._status_lock: + self._freq_snapshot = LoopFrequencyStats( + mean=stats.mean, + m2=stats.m2, + min_freq=stats.min_freq, + count=stats.count, + last_freq=stats.last_freq, + potential_freq=stats.potential_freq, + ) + + def _maybe_log_frequency(self, loop_count: int, print_interval_loops: int, stats: LoopFrequencyStats) -> None: + """Emit frequency telemetry when enough loops have elapsed.""" + if loop_count % print_interval_loops != 0 or stats.count == 0: + return + + variance = stats.m2 / stats.count if stats.count > 0 else 0.0 + lowest = stats.min_freq if stats.min_freq != float("inf") else 0.0 + logger.debug( + "Loop freq - avg: %.2fHz, variance: %.4f, min: %.2fHz, last: %.2fHz, potential: %.2fHz, target: %.1fHz", + stats.mean, + variance, + lowest, + stats.last_freq, + stats.potential_freq, + self.target_frequency, + ) + stats.reset() + + def _update_face_tracking(self, current_time: float) -> None: + """Get face tracking offsets from camera worker thread.""" + if self.camera_worker is not None: + # Get face tracking offsets from camera worker thread + offsets = self.camera_worker.get_face_tracking_offsets() + self.state.face_tracking_offsets = offsets + else: + # No camera worker, use neutral offsets + self.state.face_tracking_offsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0) + + def start(self) -> None: + """Start the worker thread that drives the 100 Hz control loop.""" + if self._thread is not None and self._thread.is_alive(): + logger.warning("Move worker already running; start() ignored") + return + self._stop_event.clear() + self._thread = threading.Thread(target=self.working_loop, daemon=True) + self._thread.start() + logger.debug("Move worker started") + + def stop(self) -> None: + """Request the worker thread to stop and wait for it to exit. + + Before stopping, resets the robot to a neutral position. + """ + if self._thread is None or not self._thread.is_alive(): + logger.debug("Move worker not running; stop() ignored") + return + + logger.info("Stopping movement manager and resetting to neutral position...") + + # Clear any queued moves and stop current move + self.clear_move_queue() + + # Stop the worker thread first so it doesn't interfere + self._stop_event.set() + if self._thread is not None: + self._thread.join() + self._thread = None + logger.debug("Move worker stopped") + + # Reset to neutral position using goto_target (same approach as wake_up) + try: + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + neutral_antennas = [0.0, 0.0] + neutral_body_yaw = 0.0 + + # Use goto_target directly on the robot + self.current_robot.goto_target( + head=neutral_head_pose, + antennas=neutral_antennas, + duration=2.0, + body_yaw=neutral_body_yaw, + ) + + logger.info("Reset to neutral position completed") + + except Exception as e: + logger.error(f"Failed to reset to neutral position: {e}") + + def get_status(self) -> Dict[str, Any]: + """Return a lightweight status snapshot for observability.""" + with self._status_lock: + pose_snapshot = clone_full_body_pose(self._last_commanded_pose) + freq_snapshot = LoopFrequencyStats( + mean=self._freq_snapshot.mean, + m2=self._freq_snapshot.m2, + min_freq=self._freq_snapshot.min_freq, + count=self._freq_snapshot.count, + last_freq=self._freq_snapshot.last_freq, + potential_freq=self._freq_snapshot.potential_freq, + ) + + head_matrix = pose_snapshot[0].tolist() if pose_snapshot else None + antennas = pose_snapshot[1] if pose_snapshot else None + body_yaw = pose_snapshot[2] if pose_snapshot else None + + return { + "queue_size": len(self.move_queue), + "is_listening": self._is_listening, + "breathing_active": self._breathing_active, + "last_commanded_pose": { + "head": head_matrix, + "antennas": antennas, + "body_yaw": body_yaw, + }, + "loop_frequency": { + "last": freq_snapshot.last_freq, + "mean": freq_snapshot.mean, + "min": freq_snapshot.min_freq, + "potential": freq_snapshot.potential_freq, + "samples": freq_snapshot.count, + }, + } + + def working_loop(self) -> None: + """Control loop main movements - reproduces main_works.py control architecture. + + Single set_target() call with pose fusion. + """ + logger.debug("Starting enhanced movement control loop (100Hz)") + + loop_count = 0 + prev_loop_start = self._now() + print_interval_loops = max(1, int(self.target_frequency * 2)) + freq_stats = self._freq_stats + + while not self._stop_event.is_set(): + loop_start = self._now() + loop_count += 1 + + if loop_count > 1: + freq_stats = self._update_frequency_stats(loop_start, prev_loop_start, freq_stats) + prev_loop_start = loop_start + + # 1) Poll external commands and apply pending offsets (atomic snapshot) + self._poll_signals(loop_start) + + # 2) Manage the primary move queue (start new move, end finished move, breathing) + self._update_primary_motion(loop_start) + + # 3) Update vision-based secondary offsets + self._update_face_tracking(loop_start) + + # 4) Build primary and secondary full-body poses, then fuse them + head, antennas, body_yaw = self._compose_full_body_pose(loop_start) + + # 5) Apply listening antenna freeze or blend-back + antennas_cmd = self._calculate_blended_antennas(antennas) + + # 6) Single set_target call - the only control point + self._issue_control_command(head, antennas_cmd, body_yaw) + + # 7) Adaptive sleep to align to next tick, then publish shared state + sleep_time, freq_stats = self._schedule_next_tick(loop_start, freq_stats) + self._publish_shared_state() + self._record_frequency_snapshot(freq_stats) + + # 8) Periodic telemetry on loop frequency + self._maybe_log_frequency(loop_count, print_interval_loops, freq_stats) + + if sleep_time > 0: + time.sleep(sleep_time) + + logger.debug("Movement control loop stopped") diff --git a/src/reachy_mini_receptionist/name_normalizer.py b/src/reachy_mini_receptionist/name_normalizer.py new file mode 100644 index 0000000000000000000000000000000000000000..f78d2e93e57049cc30683618359245e4be90884d --- /dev/null +++ b/src/reachy_mini_receptionist/name_normalizer.py @@ -0,0 +1,228 @@ +"""Gemini-backed name disambiguation. + +OpenAI's gpt-4o-transcribe mishears short non-English names (Arav -> Lee Win, +Krishna -> Christina, Mukul -> Michael). The realtime audio loop is locked +to the OpenAI stack — too risky to swap mid-pilot — but we can run the +transcribed candidate through Gemini as a cheap post-processing step to +recover the intended name. + +Pipeline: + visitor says "Arav" + -> OpenAI STT returns "Lee Win" + -> normalize_name("Lee Win", candidates=[Henry, Krishna, Arav, ...]) + -> asks Gemini: "Which of these is closest phonetically to 'Lee Win'?" + -> returns "Arav" if confident, original "Lee Win" if not + -> register_guest("Arav", confirmed=true) — saves the right face + +Fails open: when GEMINI_API_KEY is unset, the http call errors, or Gemini +returns garbage, we return the original transcribed name unchanged. The +worst case is "same behaviour as before". +""" +from __future__ import annotations + +import json +import logging +import os +from typing import Iterable, Optional + +import httpx + +logger = logging.getLogger(__name__) + +# Default to gemini-3.5-flash (Google's latest Flash-tier model, +# launched at I/O 2026 — faster and more accurate than the 2.5 +# variants on short-prompt disambiguation). Override via the +# GEMINI_MODEL env var without restarting (read on each call) so the +# operator can fall back to 2.5-flash / 2.5-flash-lite if needed. +_DEFAULT_GEMINI_MODEL = "gemini-3.5-flash" +_GEMINI_URL_TEMPLATE = ( + "https://generativelanguage.googleapis.com/v1beta/models/" + "{model}:generateContent" +) +_HTTP_TIMEOUT_SECONDS = 4.0 + + +def _gemini_url() -> str: + model = os.getenv("GEMINI_MODEL", "").strip() or _DEFAULT_GEMINI_MODEL + return _GEMINI_URL_TEMPLATE.format(model=model) + + +def _build_prompt(transcribed: str, candidates: list[str]) -> str: + cand_list = ", ".join(candidates) if candidates else "(none)" + return ( + "You are a speech-recognition error corrector for a reception desk. " + f"A visitor said their name and the speech-to-text returned: '{transcribed}'. " + f"The visitor is likely one of these scheduled or known people: {cand_list}. " + "Return a candidate name ONLY when it is very phonetically close to " + "the transcribed value — sharing most of the syllables, vowel sounds, " + "or stressed sound. If NO candidate is a clear phonetic match, you " + "MUST return the original transcribed value unchanged. Do not stretch " + "for a match. When in doubt, keep the original. " + "Good corrections (close phonetic match): " + "'Lee Win' -> 'Arav' (sounds like 'Le-win' ~ 'A-rav'? No — return 'Lee Win'). " + "'Christina' -> 'Krishna' (yes, similar syllables). " + "'Michael' -> 'Mukul' (yes, M-K consonants and similar vowels). " + "Bad corrections (NEVER do these — return the original instead): " + "'Bruh' -> stays 'Bruh' (no candidate sounds like Bruh). " + "'Bob' -> stays 'Bob' (no phonetic match in the list). " + "'Sarah' -> stays 'Sarah' (if Sarah is in the list, fine; if not, KEEP IT). " + "Return only the chosen name, no extra words, no quotes, no punctuation." + ) + + +def _phonetic_similarity(a: str, b: str) -> float: + """Cheap phonetic similarity score in [0, 1]. + + Uses Python's stdlib difflib SequenceMatcher on the lowercased + strings. Not phonetic-perfect (no Soundex / Metaphone) but good + enough to reject "Henry" -> "Arjun" type hallucinations from + Gemini. Anything below 0.5 is "not really similar". + """ + from difflib import SequenceMatcher + return SequenceMatcher(None, a.lower(), b.lower()).ratio() + + +def normalize_name( + transcribed: str, + candidates: Iterable[str], + api_key: Optional[str] = None, +) -> str: + """Map a transcribed name to the closest candidate via Gemini. + + Returns the original ``transcribed`` value unchanged when: + - GEMINI_API_KEY is unset + - the http call fails / times out / returns non-2xx + - the response is empty or obviously wrong + - Gemini's choice is not phonetically similar to the transcribed value + """ + raw = (transcribed or "").strip() + if not raw: + return raw + + # ────────────────────────────────────────────────────────────────── + # DISABLED (2026-05-21): Gemini-3.5-flash name normalizer. + # Built when OpenAI's gpt-4o-transcribe was mishearing short names + # (Arav -> Lee Win). Gemini Live hears names cleanly natively, so the + # extra REST call to gemini-3.5-flash before every register_guest / + # lookup_employee was pure latency (~1-2s per call) plus 429s on the + # free tier. Re-enable by removing this early-return when switching + # VOICE_BACKEND back to "openai". + # ────────────────────────────────────────────────────────────────── + return raw + + key = api_key or os.getenv("GEMINI_API_KEY", "").strip() + if not key: + logger.debug("normalize_name: no GEMINI_API_KEY, returning original") + return raw + + cand_list = [c for c in (candidates or []) if c and isinstance(c, str)] + if not cand_list: + return raw + + # Pre-filter: only ask Gemini about candidates with at least some + # surface similarity to the transcribed name. Cuts API cost AND + # prevents Gemini from being prompted with totally-unrelated options. + _MIN_SIMILARITY = 0.4 + similar = [c for c in cand_list if _phonetic_similarity(raw, c) >= _MIN_SIMILARITY] + if not similar: + logger.debug( + "normalize_name: no candidate similar enough to %r (best=%.2f) — keeping original", + raw, max((_phonetic_similarity(raw, c) for c in cand_list), default=0.0), + ) + return raw + cand_list = similar + + payload = { + "contents": [{ + "role": "user", + "parts": [{"text": _build_prompt(raw, cand_list)}], + }], + "generationConfig": { + "temperature": 0.0, + "maxOutputTokens": 24, + }, + } + try: + resp = httpx.post( + f"{_gemini_url()}?key={key}", + json=payload, + timeout=_HTTP_TIMEOUT_SECONDS, + ) + if resp.status_code >= 400: + logger.debug("normalize_name: gemini HTTP %d: %s", resp.status_code, resp.text[:200]) + return raw + data = resp.json() + except Exception as e: + logger.debug("normalize_name: gemini call failed (%s)", e) + return raw + + try: + candidate_text = ( + data["candidates"][0]["content"]["parts"][0]["text"] + ).strip() + except Exception: + logger.debug("normalize_name: unexpected gemini response shape: %s", json.dumps(data)[:200]) + return raw + + # Sanitize: Gemini sometimes adds punctuation despite the instruction. + for ch in ('"', "'", "."): + candidate_text = candidate_text.replace(ch, "") + candidate_text = candidate_text.strip() + if not candidate_text: + return raw + + # Only trust a Gemini reply that is either (a) exactly one of the + # candidates (case-insensitive) or (b) the original transcript. This + # prevents Gemini from hallucinating a new name we never gave it. + cand_lower = {c.lower(): c for c in cand_list} + g_lower = candidate_text.lower() + if g_lower in cand_lower: + chosen = cand_lower[g_lower] + if chosen.lower() != raw.lower(): + logger.info( + "normalize_name: '%s' -> '%s' (gemini disambiguation)", + raw, chosen, + ) + return chosen + if g_lower == raw.lower(): + return raw + logger.debug( + "normalize_name: gemini returned %r which isn't in candidates %r — keeping original %r", + candidate_text, cand_list, raw, + ) + return raw + + +def collect_known_names() -> list[str]: + """Return employee names + aliases for the candidate list. + + Used by ``register_guest`` (visitor name disambiguation against + employees, NOT against calendar visitors) and ``lookup_employee`` + (host name -> directory entries). Calendar visitor names are + excluded — including them was causing Gemini to map every + transcribed visitor name onto the next-scheduled visitor. + + Failures are silenced so a degraded source never blocks the lookup. + """ + names: list[str] = [] + try: + from reachy_mini_receptionist import employees + for emp in employees.get_all_employees(): + n = (emp.get("name") or "").strip() + if n: + names.append(n) + for alias in (emp.get("aliases") or []): + a = (alias or "").strip() + if a: + names.append(a) + except Exception: + pass + seen: set[str] = set() + unique: list[str] = [] + for n in names: + k = n.lower() + if k in seen: + continue + seen.add(k) + unique.append(n) + return unique diff --git a/src/reachy_mini_receptionist/openai_realtime.py b/src/reachy_mini_receptionist/openai_realtime.py new file mode 100644 index 0000000000000000000000000000000000000000..12dad83d0e6969501870f71db5a7a46d5027ed98 --- /dev/null +++ b/src/reachy_mini_receptionist/openai_realtime.py @@ -0,0 +1,1839 @@ +import json +import os +import uuid +import base64 +import random +import asyncio +import logging +import threading +import time +from typing import Any, Final, Tuple, Literal, Optional +from pathlib import Path +from datetime import datetime + +import cv2 +import numpy as np +import gradio as gr +from openai import AsyncOpenAI +from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item, audio_to_int16 +from numpy.typing import NDArray +from scipy.signal import resample +from websockets.exceptions import ConnectionClosedError + +from reachy_mini_receptionist.config import config +from reachy_mini_receptionist.prompts import get_session_voice, get_session_instructions +from reachy_mini_receptionist.tools.core_tools import ( + ToolDependencies, + get_tool_specs, +) +from reachy_mini_receptionist.tools.background_tool_manager import ( + ToolCallRoutine, + ToolNotification, + BackgroundToolManager, +) + + +logger = logging.getLogger(__name__) + +OPEN_AI_INPUT_SAMPLE_RATE: Final[Literal[24000]] = 24000 +OPEN_AI_OUTPUT_SAMPLE_RATE: Final[Literal[24000]] = 24000 + +# Cost tracking from usage data (pricing as of Feb 2026 https://openai.com/api/pricing/) +AUDIO_INPUT_COST_PER_1M = 32.0 +AUDIO_OUTPUT_COST_PER_1M = 64.0 +TEXT_INPUT_COST_PER_1M = 4.0 +TEXT_OUTPUT_COST_PER_1M = 16.0 +IMAGE_INPUT_COST_PER_1M = 5.0 + +_RESPONSE_DONE_TIMEOUT: Final[float] = 30.0 + +# How often to rebuild the STT bias prompt + push it to the realtime session +# so calendar additions made AFTER the session connected (visitors added on +# the fly) still benefit from name-biased transcription. 300s aligns with +# the iCal cache TTL so each refresh either reuses the cache or triggers +# at most one fetch. +_STT_BIAS_REFRESH_SECONDS: Final[float] = 300.0 + + +def _format_bias_prompt(names: list[str]) -> str: + seen: set[str] = set() + unique: list[str] = [] + for n in names: + key = n.lower() + if key in seen: + continue + seen.add(key) + unique.append(n) + + if not unique: + return ( + "Reception lobby check-in conversation. Visitor names and host names." + ) + + # Plain comma-separated bias list. We tried beefing this up with + # "I am . Here to see ." sentences per name to increase + # the bias signal — but gpt-4o-transcribe started ECHOING those + # sentences back as the user transcript when audio was unclear, + # so real visitor speech ("I am David, here to see Andrew") was + # being mistranscribed as "I am David. Here to see David." (a + # name from the prompt copied to both slots). Going back to the + # simple list keeps the bias direction without the echo failure. + # + # OpenAI's Realtime API caps the transcription prompt at 1024 chars. + _MAX_PROMPT_CHARS = 1000 + body = ", ".join(unique) + msg = ( + "Reception lobby check-in. Expected visitor and host names include: " + + body + + "." + ) + if len(msg) > _MAX_PROMPT_CHARS: + msg = msg[: _MAX_PROMPT_CHARS - 1] + "." + return msg + + +def _collect_employee_names() -> list[str]: + names: list[str] = [] + try: + from reachy_mini_receptionist import employees + for emp in employees.get_all_employees(): + n = (emp.get("name") if isinstance(emp, dict) else getattr(emp, "name", "")) or "" + n = n.strip() + if n: + names.append(n) + aliases = ( + emp.get("aliases") if isinstance(emp, dict) else getattr(emp, "aliases", None) + ) or [] + for alias in aliases: + a = (alias or "").strip() + if a: + names.append(a) + except Exception: + pass + return names + + +def _collect_appointment_names_sync() -> list[str]: + # See _collect_appointment_names_async — calendar visitor names are + # intentionally excluded from STT bias to stop the model defaulting + # every short utterance to the next scheduled visitor. + return [] + + +async def _collect_appointment_names_async() -> list[str]: + # Calendar visitor names are intentionally NOT included in the STT + # bias prompt. We observed STT picking "Henry" (the next calendar + # entry) for any short utterance from a visitor, because the bias + # heavily prefers names that appear in the prompt. The employee + # directory provides enough name coverage for hosts; visitor names + # have to be heard fresh from speech. + return [] + + +def _build_transcription_bias_prompt() -> str: + """Sync variant — kept for callers outside an event loop. Blocks on iCal + HTTP. Prefer ``_build_transcription_bias_prompt_async`` in async paths. + """ + return _format_bias_prompt(_collect_appointment_names_sync() + _collect_employee_names()) + + +async def _build_transcription_bias_prompt_async() -> str: + """Assemble the STT bias prompt without blocking the event loop on iCal. + + OpenAI's transcription API accepts a free-form ``prompt`` string. When + the recognizer hears mumbled audio it leans toward words/phrases in this + prompt. Feeding today's calendar visitors + the employee directory + makes non-English names (Mukul, Krishna, Shyam, etc.) far more likely + to come back correct instead of collapsing to "Michael"/"Christina". + + Fails open — if a source raises, the generic prompt is returned. + """ + return _format_bias_prompt( + await _collect_appointment_names_async() + _collect_employee_names() + ) + + +def _compute_response_cost(usage: Any) -> float: + """Compute dollar cost from a response usage object.""" + inp = getattr(usage, "input_token_details", None) + out = getattr(usage, "output_token_details", None) + cost = 0.0 + if inp: + cost += (getattr(inp, "audio_tokens", 0) or 0) * AUDIO_INPUT_COST_PER_1M / 1e6 + cost += (getattr(inp, "text_tokens", 0) or 0) * TEXT_INPUT_COST_PER_1M / 1e6 + cost += (getattr(inp, "image_tokens", 0) or 0) * IMAGE_INPUT_COST_PER_1M / 1e6 + if out: + cost += (getattr(out, "audio_tokens", 0) or 0) * AUDIO_OUTPUT_COST_PER_1M / 1e6 + cost += (getattr(out, "text_tokens", 0) or 0) * TEXT_OUTPUT_COST_PER_1M / 1e6 + return cost + + +class OpenaiRealtimeHandler(AsyncStreamHandler): + """An OpenAI realtime handler for fastrtc Stream.""" + + def __init__( + self, + deps: ToolDependencies, + gradio_mode: bool = False, + instance_path: Optional[str] = None, + session_manager: Any | None = None, + controller: Any | None = None, + ): + """Initialize the handler. + + ``session_manager`` and ``controller`` are optional so existing test + harnesses that construct the handler directly keep working. + """ + super().__init__( + expected_layout="mono", + output_sample_rate=OPEN_AI_OUTPUT_SAMPLE_RATE, + input_sample_rate=OPEN_AI_INPUT_SAMPLE_RATE, + ) + + # Override typing of the sample rates to match OpenAI's requirements + self.output_sample_rate: Literal[24000] = self.output_sample_rate + self.input_sample_rate: Literal[24000] = self.input_sample_rate + + self.deps = deps + self._session_manager = session_manager + self._controller = controller + + # Override type annotations for OpenAI strict typing (only for values used in API) + self.output_sample_rate = OPEN_AI_OUTPUT_SAMPLE_RATE + self.input_sample_rate = OPEN_AI_INPUT_SAMPLE_RATE + + self.connection: Any = None + self.output_queue: "asyncio.Queue[Tuple[int, NDArray[np.int16]] | AdditionalOutputs]" = asyncio.Queue() + + self.last_activity_time = asyncio.get_event_loop().time() + self.start_time = asyncio.get_event_loop().time() + self.is_idle_tool_call = False + self.gradio_mode = gradio_mode + self.instance_path = instance_path + # Track how the API key was provided (env vs textbox) and its value + self._key_source: Literal["env", "textbox"] = "env" + self._provided_api_key: str | None = None + + # Debouncing for partial transcripts + self.partial_transcript_task: asyncio.Task[None] | None = None + self.partial_transcript_sequence: int = 0 # sequence counter to prevent stale emissions + self.partial_debounce_delay = 0.5 # seconds + + # Internal lifecycle flags + self._shutdown_requested: bool = False + self._connected_event: asyncio.Event = asyncio.Event() + + # Background tool manager + self.tool_manager = BackgroundToolManager() + + # Cost tracking + self.cumulative_cost: float = 0.0 + + # Response-in-progress guard: the Realtime API only allows one active + # response per conversation at a time. A dedicated worker task + # (_response_sender_loop) dequeues and sends one request at a time + self._pending_responses: asyncio.Queue[dict[str, Any]] = asyncio.Queue() + self._response_done_event: asyncio.Event = asyncio.Event() + self._response_done_event.set() + self._last_response_rejected: bool = False + self._runtime_loop: asyncio.AbstractEventLoop | None = None + + # Last successfully pushed external face context event. + self._face_event_lock = threading.Lock() + self._last_face_event_sent: dict[str, Any] | None = None + # Last pending external face event waiting for runtime loop/connection. + self._pending_face_event_lock = threading.Lock() + self._pending_face_event: dict[str, Any] | None = None + + # Tool args by call_id, populated when a tool starts and consumed + # when its result arrives (so the ConversationController gets both + # the args and the result). + self._tool_call_args: dict[str, dict[str, Any]] = {} + + # Last successfully pushed session context event + pending buffer. + self._session_event_lock = threading.Lock() + self._last_session_event_sent: dict[str, Any] | None = None + self._pending_session_event_lock = threading.Lock() + self._pending_session_event: dict[str, Any] | None = None + + # One-shot guard so the IDLE-state workflow hint is only pushed + # once per IDLE stretch, not on every speech_started event. + self._idle_speech_cue_pushed: bool = False + + # Last STT bias prompt actually sent to the realtime session, so the + # periodic refresh loop can skip the session.update when nothing has + # changed (calendar quiet, employee directory unchanged). + self._last_stt_bias_prompt: Optional[str] = None + + def _stash_pending_face_event(self, face_event: dict[str, Any]) -> None: + """Keep only the latest pending face event for eventual delivery.""" + with self._pending_face_event_lock: + self._pending_face_event = dict(face_event) + + def _pop_pending_face_event(self) -> dict[str, Any] | None: + """Pop and clear latest pending face event.""" + with self._pending_face_event_lock: + pending = self._pending_face_event + self._pending_face_event = None + return pending + + async def _flush_pending_face_event(self) -> None: + """Try sending one buffered face event once the session is ready.""" + pending = self._pop_pending_face_event() + if pending is None: + return + try: + await self._push_face_context_event(pending) + except Exception as e: + logger.debug("Failed to flush pending face event: %s", e) + self._stash_pending_face_event(pending) + + def copy(self) -> "OpenaiRealtimeHandler": + """Create a copy of the handler.""" + return OpenaiRealtimeHandler( + self.deps, + self.gradio_mode, + self.instance_path, + session_manager=self._session_manager, + controller=self._controller, + ) + + async def apply_personality(self, profile: str | None) -> str: + """Apply a new personality (profile) at runtime if possible. + + - Updates the global config's selected profile for subsequent calls. + - If a realtime connection is active, sends a session.update with the + freshly resolved instructions so the change takes effect immediately. + + Returns a short status message for UI feedback. + """ + try: + # Update the in-process config value and env + from reachy_mini_receptionist.config import config as _config + from reachy_mini_receptionist.config import set_custom_profile + + set_custom_profile(profile) + logger.info( + "Set custom profile to %r (config=%r)", profile, getattr(_config, "REACHY_MINI_CUSTOM_PROFILE", None) + ) + + try: + instructions = get_session_instructions() + voice = get_session_voice() + except BaseException as e: # catch SystemExit from prompt loader without crashing + logger.error("Failed to resolve personality content: %s", e) + return f"Failed to apply personality: {e}" + + # Attempt a live update first, then force a full restart to ensure it sticks + if self.connection is not None: + try: + await self.connection.session.update( + session={ + "type": "realtime", + "instructions": instructions, + "audio": {"output": {"voice": voice}}, + }, + ) + logger.info("Applied personality via live update: %s", profile or "built-in default") + except Exception as e: + logger.warning("Live update failed; will restart session: %s", e) + + # Force a real restart to guarantee the new instructions/voice + try: + await self._restart_session() + return "Applied personality and restarted realtime session." + except Exception as e: + logger.warning("Failed to restart session after apply: %s", e) + return "Applied personality. Will take effect on next connection." + else: + logger.info( + "Applied personality recorded: %s (no live connection; will apply on next session)", + profile or "built-in default", + ) + return "Applied personality. Will take effect on next connection." + except Exception as e: + logger.error("Error applying personality '%s': %s", profile, e) + return f"Failed to apply personality: {e}" + + async def _emit_debounced_partial(self, transcript: str, sequence: int) -> None: + """Emit partial transcript after debounce delay.""" + try: + await asyncio.sleep(self.partial_debounce_delay) + # Only emit if this is still the latest partial (by sequence number) + if self.partial_transcript_sequence == sequence: + await self.output_queue.put(AdditionalOutputs({"role": "user_partial", "content": transcript})) + logger.debug(f"Debounced partial emitted: {transcript}") + except asyncio.CancelledError: + logger.debug("Debounced partial cancelled") + raise + + async def start_up(self) -> None: + """Start the handler with minimal retries on unexpected websocket closure.""" + self._runtime_loop = asyncio.get_running_loop() + openai_api_key = config.OPENAI_API_KEY + if self.gradio_mode and not openai_api_key: + # api key was not found in .env or in the environment variables + await self.wait_for_args() # type: ignore[no-untyped-call] + args = list(self.latest_args) + textbox_api_key = args[3] if len(args[3]) > 0 else None + if textbox_api_key is not None: + openai_api_key = textbox_api_key + self._key_source = "textbox" + self._provided_api_key = textbox_api_key + else: + openai_api_key = config.OPENAI_API_KEY + else: + if not openai_api_key or not openai_api_key.strip(): + # In headless console mode, LocalStream now blocks startup until the key is provided. + # However, unit tests may invoke this handler directly with a stubbed client. + # To keep tests hermetic without requiring a real key, fall back to a placeholder. + logger.warning("OPENAI_API_KEY missing. Proceeding with a placeholder (tests/offline).") + openai_api_key = "DUMMY" + + self.client = AsyncOpenAI(api_key=openai_api_key) + + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + try: + await self._run_realtime_session() + # Normal exit from the session, stop retrying + return + except ConnectionClosedError as e: + # Abrupt close (e.g., "no close frame received or sent") → retry + logger.warning("Realtime websocket closed unexpectedly (attempt %d/%d): %s", attempt, max_attempts, e) + if attempt < max_attempts: + # exponential backoff with jitter + base_delay = 2 ** (attempt - 1) # 1s, 2s, 4s, 8s, etc. + jitter = random.uniform(0, 0.5) + delay = base_delay + jitter + logger.info("Retrying in %.1f seconds...", delay) + await asyncio.sleep(delay) + continue + raise + finally: + # never keep a stale reference + self.connection = None + try: + self._connected_event.clear() + except Exception: + pass + + def notify_external_face_event(self, face_event: dict[str, Any]) -> None: + """Thread-safe entrypoint for face worker state transition events. + + This injects context into the conversation via conversation.item.create + without forcing a response. + """ + loop = self._runtime_loop + if loop is None or loop.is_closed(): + logger.debug("Deferring face event (runtime loop not ready): %s", face_event) + self._stash_pending_face_event(face_event) + return + + try: + future = asyncio.run_coroutine_threadsafe(self._push_face_context_event(face_event), loop) + + def _on_done(fut: "asyncio.Future[None]") -> None: + try: + fut.result() + except Exception as e: + logger.debug("Face context push failed: %s", e) + self._stash_pending_face_event(face_event) + + future.add_done_callback(_on_done) + except Exception as e: + logger.debug("Failed to schedule face context event: %s", e) + self._stash_pending_face_event(face_event) + + async def _push_face_context_event(self, face_event: dict[str, Any]) -> None: + """Push a face state change as external context without triggering a response.""" + if not self.connection: + logger.debug("Deferring face context event (no connection): %s", face_event) + self._stash_pending_face_event(face_event) + return + + state = str(face_event.get("state", "unknown")) + name = face_event.get("name") + previous_state = str(face_event.get("previous_state", "unknown")) + previous_name = face_event.get("previous_name") + lbph_conf = face_event.get("lbph_confidence", 0.0) + det_conf = face_event.get("detection_confidence", 0.0) + + msg = ( + f"[External face update {self.format_timestamp()}] " + f"state={state}; name={name}; previous_state={previous_state}; " + f"previous_name={previous_name}; lbph={lbph_conf}; det={det_conf}. " + "This is context only. Do not respond unless the user speaks." + ) + + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": msg}], + }, + ) + + sent_at_epoch = time.time() + sent_payload = { + "state": state, + "name": name, + "previous_state": previous_state, + "previous_name": previous_name, + "lbph_confidence": float(lbph_conf), + "detection_confidence": float(det_conf), + "sent_at": sent_at_epoch, + "sent_at_iso": datetime.fromtimestamp(sent_at_epoch).strftime("%Y-%m-%d %H:%M:%S"), + } + with self._face_event_lock: + self._last_face_event_sent = sent_payload + + logger.info("Pushed external face context event: %s", msg) + + async def _prime_no_face_context(self) -> None: + """Prime a fresh session with an explicit no-face context event.""" + try: + await self._push_face_context_event( + { + "event": "face_state_changed", + "state": "no_face", + "name": None, + "previous_state": "unknown", + "previous_name": None, + "lbph_confidence": 0.0, + "detection_confidence": 0.0, + "timestamp": time.time(), + } + ) + logger.info("Primed startup face context: state=no_face") + except Exception as e: + logger.debug("Failed to prime startup no-face context: %s", e) + + def get_last_face_event_sent(self) -> dict[str, Any] | None: + """Return the last face context event that was successfully sent to the model.""" + with self._face_event_lock: + if self._last_face_event_sent is None: + return None + return dict(self._last_face_event_sent) + + async def _push_idle_speech_cue_if_needed(self) -> None: + """Push the IDLE-state workflow hint when a visitor speaks first. + + Per-state hints normally arrive via ``notify_session_event`` on + transitions. While state stays IDLE no transition fires, so the + LLM has no guidance when the camera hasn't caught the visitor's + face yet. We rate-limit to once per IDLE stretch so we don't + push the same cue on every breath. + """ + if not self.connection: + return + if self._session_manager is None: + return + try: + from reachy_mini_receptionist.receptionist_state import ReceptionState + from reachy_mini_receptionist.conversation_controller import next_action_hint + current = self._session_manager.current_state + if current != ReceptionState.IDLE: + return + hint = next_action_hint(current) + if not hint: + return + if getattr(self, "_idle_speech_cue_pushed", False): + return + msg = ( + f"[Backend idle-speech cue {self.format_timestamp()}] " + f"Visitor just started speaking while state=idle. {hint}" + ) + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": msg}], + }, + ) + self._idle_speech_cue_pushed = True + logger.info("Pushed idle-speech cue: %s", msg) + except Exception as e: + logger.debug("idle-speech cue push error: %s", e) + + # ------------------------------------------------------------------ + # Session context events (mirror of the face-event channel) + # ------------------------------------------------------------------ + + def _stash_pending_session_event(self, payload: dict[str, Any]) -> None: + with self._pending_session_event_lock: + self._pending_session_event = dict(payload) + + def _pop_pending_session_event(self) -> dict[str, Any] | None: + with self._pending_session_event_lock: + pending = self._pending_session_event + self._pending_session_event = None + return pending + + async def _flush_pending_session_event(self) -> None: + pending = self._pop_pending_session_event() + if pending is None: + return + try: + await self._push_session_context_event(pending) + except Exception as e: + logger.debug("Failed to flush pending session event: %s", e) + self._stash_pending_session_event(pending) + + def notify_session_event( + self, + previous_state: Any, + new_state: Any, + snapshot: Any, + ) -> None: + """Subscriber callback for SessionManager — thread-safe. + + Schedules a coroutine on the runtime loop that pushes the session + state change to the LLM as a context-only conversation item. + """ + try: + payload = { + "previous_state": getattr(previous_state, "value", str(previous_state)), + "new_state": getattr(new_state, "value", str(new_state)), + "snapshot": snapshot.to_dict() if hasattr(snapshot, "to_dict") else {}, + } + except Exception as e: + logger.debug("notify_session_event: failed to build payload: %s", e) + return + + # Reset the idle-speech cue flag whenever the session transitions + # back to IDLE (visitor walked away, session reset, timeout). The + # next visitor's first utterance will re-push the cue. + try: + new_state_value = payload["new_state"] + if new_state_value == "idle": + self._idle_speech_cue_pushed = False + except Exception: + pass + + loop = self._runtime_loop + if loop is None or loop.is_closed(): + logger.debug("Deferring session event (runtime loop not ready): %s", payload) + self._stash_pending_session_event(payload) + return + + try: + future = asyncio.run_coroutine_threadsafe( + self._push_session_context_event(payload), loop + ) + + def _on_done(fut: "asyncio.Future[None]") -> None: + try: + fut.result() + except Exception as e: + logger.debug("Session context push failed: %s", e) + self._stash_pending_session_event(payload) + + future.add_done_callback(_on_done) + except Exception as e: + logger.debug("Failed to schedule session context event: %s", e) + self._stash_pending_session_event(payload) + + async def _push_session_context_event(self, payload: dict[str, Any]) -> None: + """Push a session state change as context-only conversation item. + + Includes a per-state ``Next:`` directive (from + ``conversation_controller.next_action_hint``) so the LLM knows what + to do when the visitor next speaks — without that workflow being + baked into the system prompt. + """ + if not self.connection: + logger.debug("Deferring session context event (no connection): %s", payload) + self._stash_pending_session_event(payload) + return + + snap = payload.get("snapshot") or {} + new_state_value = payload.get("new_state") + + hint = "" + speak_now = False + try: + from reachy_mini_receptionist.conversation_controller import ( + next_action_hint, + should_speak_immediately, + ) + from reachy_mini_receptionist.receptionist_state import ReceptionState + if new_state_value: + new_state_enum = ReceptionState(new_state_value) + hint = next_action_hint(new_state_enum) + speak_now = should_speak_immediately(new_state_enum) + except Exception as e: + logger.debug("Could not compute next_action_hint: %s", e) + + base = ( + f"[Backend session update {self.format_timestamp()}] " + f"state: {payload.get('previous_state')} -> {new_state_value}; " + f"visitor={snap.get('visitor_name')}; " + f"employee={snap.get('employee_name')}; " + f"appointment={(snap.get('matched_appointment') or {}).get('time')}; " + f"email_sent_to={snap.get('email_sent_to')}." + ) + + if hint and speak_now: + # The visitor is waiting for the bot to finish a sequence it + # started. Tell the LLM to speak now — no "stay quiet" suffix. + msg = f"{base} SPEAK NOW: {hint}" + elif hint: + # State change happened passively (face event, etc.) — the bot + # should not blurt anything; act on the hint when the visitor + # next speaks. + msg = ( + f"{base} Next: {hint} " + "(Stay quiet until the visitor speaks; this is context only.)" + ) + else: + msg = f"{base} This is context only. Do not respond unless the user speaks." + + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": msg}], + }, + ) + + sent_payload = {**payload, "sent_at": time.time(), "hint": hint} + with self._session_event_lock: + self._last_session_event_sent = sent_payload + + logger.info("Pushed session context event: %s", msg) + + # For speak-now transitions, actively trigger a response so the + # LLM speaks immediately. Face-driven transitions to RECOGNIZED + # have no in-flight response cycle to piggyback on, and even + # tool-driven transitions previously stalled when the SPEAK NOW + # cue conflicted with the older "stay quiet" suffix. The sender + # worker serializes any duplicate response.create with the + # tool-result handler's call. + if speak_now: + try: + await self._safe_response_create( + response={ + "instructions": ( + "Use the latest [Backend session update] context " + "and speak to the visitor now. Keep it concise." + ), + }, + ) + except Exception as e: + logger.debug("Failed to queue speak-now response.create: %s", e) + + def get_last_session_event_sent(self) -> dict[str, Any] | None: + """Return the last session context event sent to the model.""" + with self._session_event_lock: + if self._last_session_event_sent is None: + return None + return dict(self._last_session_event_sent) + + async def _restart_session(self) -> None: + """Force-close the current session and start a fresh one in background. + + Does not block the caller while the new session is establishing. + """ + try: + if self.connection is not None: + try: + await self.connection.close() + except Exception: + pass + finally: + self.connection = None + + # Ensure we have a client (start_up must have run once) + if getattr(self, "client", None) is None: + logger.warning("Cannot restart: OpenAI client not initialized yet.") + return + + # Fire-and-forget new session and wait briefly for connection + try: + self._connected_event.clear() + except Exception: + pass + asyncio.create_task(self._run_realtime_session(), name="openai-realtime-restart") + try: + await asyncio.wait_for(self._connected_event.wait(), timeout=5.0) + logger.info("Realtime session restarted and connected.") + except asyncio.TimeoutError: + logger.warning("Realtime session restart timed out; continuing in background.") + except Exception as e: + logger.warning("_restart_session failed: %s", e) + + async def _safe_response_create(self, **kwargs: Any) -> None: + """Enqueue a response.create() kwargs for the sender worker _response_sender_loop(). + + This method never blocks the caller. + """ + await self._pending_responses.put(kwargs) + + async def _stt_bias_refresh_loop(self) -> None: + """DISABLED for the demo cut. The periodic session.update was + suspected of breaking server VAD mid-session (yes/no replies + stopped being captured). Re-enable after the demo with more + testing. + """ + return + try: # noqa: unreachable until re-enabled + while self.connection: + try: + await asyncio.sleep(_STT_BIAS_REFRESH_SECONDS) + except asyncio.CancelledError: + return + + if not self.connection: + return + + try: + prompt = await _build_transcription_bias_prompt_async() + except Exception as e: + logger.debug("STT bias refresh: build failed: %s", e) + continue + + if prompt == self._last_stt_bias_prompt: + continue + + try: + # Re-send both transcription AND turn_detection together. + # The realtime API treats nested updates as full + # replacement of the parent object on some server + # versions — sending only `transcription` could reset + # `turn_detection` back to stock defaults (threshold + # 0.5 / silence 500ms) and silently change VAD behavior + # mid-conversation. Pinning both keeps VAD consistent. + await self.connection.session.update( + session={ + "type": "realtime", + "audio": { + "input": { + "transcription": { + "model": "gpt-4o-transcribe", + "language": "en", + "prompt": prompt, + }, + "turn_detection": { + "type": "server_vad", + "threshold": 0.5, + "silence_duration_ms": 600, + "prefix_padding_ms": 300, + "interrupt_response": True, + "create_response": True, + }, + }, + }, + }, + ) + self._last_stt_bias_prompt = prompt + logger.info( + "STT bias refreshed (%d chars) — calendar/directory change detected", + len(prompt), + ) + except Exception as e: + logger.debug("STT bias refresh: session.update failed: %s", e) + except asyncio.CancelledError: + return + + def _just_entered_speak_now_state(self) -> bool: + """True if the session is currently in a SPEAK_NOW state. + + Used in ``_handle_tool_result`` to skip the generic post-tool + narration when a controller-driven SPEAK_NOW transition has already + enqueued a state-specific ``response.create``. + """ + if self._session_manager is None: + return False + try: + from reachy_mini_receptionist.conversation_controller import should_speak_immediately + return bool(should_speak_immediately(self._session_manager.current_state)) + except Exception: + return False + + async def _response_sender_loop(self) -> None: + """Dedicated worker that sends ``response.create()`` calls serially. + + This logic was designed to comply with the response.create() docstring specification for event ordering: + https://github.com/openai/openai-python/blob/3e0c05b84a2056870abf3bd6a5e7849020209cc3/src/openai/resources/realtime/realtime.py#L649C1-L651C30 + + For each queued request the worker: + 1. Waits until no response is active (_response_done_event). + 2. Sends response.create(). + 3. Waits for the response cycle to complete (response.done). + 4. If the server rejected with active_response, retries from step 1. + """ + while self.connection: + try: + kwargs = await self._pending_responses.get() + except asyncio.CancelledError: + return + + sent = False + max_retries = 5 + attempts = 0 + while not sent and self.connection and attempts < max_retries: + try: + await asyncio.wait_for(self._response_done_event.wait(), timeout=_RESPONSE_DONE_TIMEOUT) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for previous response to finish; forcing ahead") + self._response_done_event.set() + + if not self.connection: + break + + self._last_response_rejected = False + try: + await self.connection.response.create(**kwargs) + except Exception as e: + logger.debug("_response_sender_loop: send failed: %s", e) + self._response_done_event.set() + break + + try: + await asyncio.wait_for(self._response_done_event.wait(), timeout=_RESPONSE_DONE_TIMEOUT) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for response.done; assuming response completed") + self._response_done_event.set() + break + + # Check if we were rejected + if self._last_response_rejected: + attempts += 1 + if attempts >= max_retries: + logger.debug("response.create rejected %d times; giving up", attempts) + break + logger.debug("response.create was rejected; retrying (%d/%d)", attempts, max_retries) + continue + + sent = True + + async def _handle_tool_result(self, bg_tool: ToolNotification) -> None: + """Process the result of a tool call.""" + if bg_tool.error is not None: + logger.error("Tool '%s' (id=%s) failed with error: %s", bg_tool.tool_name, bg_tool.id, bg_tool.error) + tool_result = {"error": bg_tool.error} + elif bg_tool.result is not None: + tool_result = bg_tool.result + logger.info( + "Tool '%s' (id=%s) executed successfully.", + bg_tool.tool_name, bg_tool.id, + ) + logger.debug("Tool '%s' full result: %s", bg_tool.tool_name, tool_result) + else: + logger.warning("Tool '%s' (id=%s) returned no result and no error", bg_tool.tool_name, bg_tool.id) + tool_result = {"error": "No result returned from tool execution"} + + call_args = self._tool_call_args.pop(bg_tool.id, {}) + + # Connection may have closed while tool was running + if not self.connection: + logger.warning("Connection closed during tool '%s' (id=%s) execution; cannot send result back", bg_tool.tool_name, bg_tool.id) + # Even if we can't send the function_call_output, the controller + # still needs to advance state so the dashboard reflects reality + # and a future reconnect lands in the right state. + if self._controller is not None: + try: + await self._controller.on_tool_completed_async( + bg_tool.tool_name, call_args, tool_result, + ) + except Exception as e: + logger.warning( + "ConversationController.on_tool_completed_async raised %s: %s", + type(e).__name__, e, + ) + return + + try: + # Send the tool result back to the model FIRST. The controller + # callback below may push a SPEAK_NOW context event + enqueue a + # response.create — if that ``response.create`` reaches the + # server before this ``function_call_output``, the model + # generates a response without seeing the tool result. Order + # matters: function_call_output → controller → context push → + # response.create. + if isinstance(bg_tool.id, str): + await self.connection.conversation.item.create( + item={ + "type": "function_call_output", + "call_id": bg_tool.id, + "output": json.dumps(tool_result), + }, + ) + + # Notify the conversation controller about the completion so it + # can transition the session. Now that the function_call_output + # is in flight ahead of us, any SPEAK_NOW response.create enqueued + # by the resulting transition will be ordered correctly. + if self._controller is not None: + try: + await self._controller.on_tool_completed_async( + bg_tool.tool_name, call_args, tool_result, + ) + except Exception as e: + logger.warning( + "ConversationController.on_tool_completed_async raised %s: %s", + type(e).__name__, e, + ) + + await self.output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": json.dumps(tool_result), + # Gradio UI metadata.status accept only "pending" and "done". Do not accept bg.tool.status values. + "metadata": { + "title": f"🛠️ Used tool {bg_tool.tool_name}", + "status": "done", + }, + }, + ), + ) + + if bg_tool.tool_name == "camera" and "b64_im" in tool_result: + # use raw base64, don't json.dumps (which adds quotes) + b64_im = tool_result["b64_im"] + if not isinstance(b64_im, str): + logger.warning("Unexpected type for b64_im: %s", type(b64_im)) + b64_im = str(b64_im) + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": f"data:image/jpeg;base64,{b64_im}", + }, + ], + }, + ) + logger.info("Added camera image to conversation") + + if self.deps.camera_worker is not None: + np_img = self.deps.camera_worker.get_latest_frame() + if np_img is not None: + # Camera frames are BGR from OpenCV; convert so Gradio displays correct colors. + rgb_frame = cv2.cvtColor(np_img, cv2.COLOR_BGR2RGB) + else: + rgb_frame = None + img = gr.Image(value=rgb_frame) + + await self.output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": img, + }, + ), + ) + + # If this tool call was triggered by an idle signal, don't make the robot speak. + # For other tool calls, let the robot reply out loud — UNLESS the + # controller has just driven the session into a SPEAK_NOW state + # (RECOGNIZED, APPOINTMENT_MATCHED, NO_APPOINTMENT, NOTIFIED, …). + # Those transitions already enqueued a state-specific + # ``response.create`` via the session-event push, and stacking + # the generic "Use the tool result and answer concisely" on top + # makes the bot speak twice in a row. + if not bg_tool.is_idle_tool_call and not self._just_entered_speak_now_state(): + # If the tool was BLOCKED / refused (success=False with a + # blocked_reason), the visitor doesn't know — bot must + # speak the friendly version of the error out loud, + # not just sit on the rejection. + blocked_reason = None + if isinstance(tool_result, dict): + blocked_reason = tool_result.get("blocked_reason") + if blocked_reason: + instructions = ( + "The last tool call was rejected by the backend. " + "Tell the visitor briefly and naturally what to do " + "next based on the error message you just received — " + "for example, ask their name again, ask for " + "confirmation, or offer a numbered choice of similar " + "names. ALWAYS speak; never go silent after a tool " + "error." + ) + else: + instructions = "Use the tool result just returned and answer concisely in speech." + await self._safe_response_create( + response={"instructions": instructions}, + ) + + # Re-synchronize the head wobble after a tool call that may have taken some time + if self.deps.head_wobbler is not None: + self.deps.head_wobbler.reset() + + except ConnectionClosedError: + logger.warning("Connection closed while sending tool result") + self.connection = None + self._response_done_event.set() + + async def _run_realtime_session(self) -> None: + """Establish and manage a single realtime session.""" + async with self.client.realtime.connect(model=config.MODEL_NAME) as conn: + # Build the transcription block. We attempt the rich form first + # (language="en" + bias prompt of expected names) and fall back + # to a minimal {"model": ...} form if the realtime API rejects + # any of the optional fields. Losing the bias prompt is a + # quality regression, not a correctness one — but losing the + # whole session.update would mean NO tools / NO instructions. + # STT_MODEL env var lets us switch between gpt-4o-transcribe + # (default, fast but hallucinates) and whisper-1 (slower, + # more conservative, returns "" on uncertainty which our + # pipeline handles cleanly). + stt_model = (os.getenv("STT_MODEL") or "gpt-4o-transcribe").strip() or "gpt-4o-transcribe" + + # STT_DISABLE_BIAS=1 turns off the name bias prompt entirely. + # The bias prompt is what causes gpt-4o-transcribe to echo + # back its own prompt as the user transcript; disabling it + # trades some name-accuracy for no echo bug at all. + bias_disabled = (os.getenv("STT_DISABLE_BIAS") or "").strip().lower() in {"1", "true", "yes"} + + initial_bias_prompt = ( + "" if bias_disabled else await _build_transcription_bias_prompt_async() + ) + transcription_full = { + "model": stt_model, + "language": "en", + } + if initial_bias_prompt: + transcription_full["prompt"] = initial_bias_prompt + logger.info( + "STT config: model=%s bias_chars=%d", + stt_model, len(initial_bias_prompt), + ) + transcription_min = {"model": "gpt-4o-transcribe"} + + def _build_session(transcription: dict[str, Any]) -> dict[str, Any]: + return { + "type": "realtime", + # NOTE: "language" is NOT a valid top-level Realtime API + # session field (it lives under audio.input.transcription). + # Output language is controlled via the instructions prompt + # (the locked profile already says "You ONLY speak ENGLISH"). + "instructions": get_session_instructions(), + "audio": { + "input": { + "format": { + "type": "audio/pcm", + "rate": self.input_sample_rate, + }, + "transcription": transcription, + # Lobby-tuned VAD: defaults (threshold 0.5, + # silence 500ms) are too aggressive — ambient + # noise interrupts the bot mid-sentence and + # hesitant speakers get cut off before + # finishing. Bump threshold + silence so the + # bot waits for a clear, complete utterance. + "turn_detection": { + "type": "server_vad", + "threshold": 0.5, + "silence_duration_ms": 500, + "prefix_padding_ms": 300, + # interrupt_response=true was causing the + # mic+VAD path to lock up when the bot's + # own audio bled into the mic — server VAD + # never saw a clean silence to commit the + # visitor's "yes". Off is safer for a + # speakerphone lobby setup. + "interrupt_response": False, + "create_response": True, + }, + }, + "output": { + "format": { + "type": "audio/pcm", + "rate": self.output_sample_rate, + }, + "voice": get_session_voice(), + }, + }, + "tools": get_tool_specs(), # type: ignore[typeddict-item] + "tool_choice": "auto", + } + + try: + try: + await conn.session.update(session=_build_session(transcription_full)) + self._last_stt_bias_prompt = initial_bias_prompt + logger.info( + "Realtime session: applied gpt-4o-transcribe with language=en + name bias (%d chars)", + len(transcription_full["prompt"]), + ) + except Exception as e: + logger.warning( + "Realtime session.update rejected the rich transcription block " + "(language/prompt) — retrying without them: %s", e, + ) + await conn.session.update(session=_build_session(transcription_min)) + self._last_stt_bias_prompt = None + logger.info("Realtime session: applied gpt-4o-transcribe (minimal fallback)") + logger.info( + "Realtime session initialized with profile=%r voice=%r", + getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None), + get_session_voice(), + ) + # If we reached here, the session update succeeded which implies the API key worked. + # Persist the key to a newly created .env (copied from .env.example) if needed. + self._persist_api_key_if_needed() + except Exception as e: + # A failed session.update means NO instructions, NO tools, NO profile will be + # active — the model will run as a generic assistant. Always log clearly. + logger.exception( + "Realtime session.update failed — robot will use DEFAULT personality with NO tools. " + "Check the session dict for invalid fields. Error: %s", e + ) + return + + logger.info("Realtime session updated successfully") + + # Manage event received from the openai server + self.connection = conn + # Reset the idle timer NOW (when the session is actually live) rather than + # at __init__ time. If connection setup takes >15 s, the idle timer would + # otherwise already be expired on the very first emit() call, causing the + # model to speak immediately without being addressed. + self.last_activity_time = asyncio.get_event_loop().time() + try: + self._connected_event.set() + except Exception: + pass + self._runtime_loop = asyncio.get_running_loop() + await self._prime_no_face_context() + await self._flush_pending_face_event() + + # Subscribe to session-state changes so transitions get pushed + # to the LLM as context events. Idempotent — subscribe() replaces + # the existing callback if any. + if self._session_manager is not None: + try: + self._session_manager.subscribe(self.notify_session_event) + except Exception as e: + logger.debug("Failed to subscribe to SessionManager: %s", e) + await self._flush_pending_session_event() + + + response_sender_task: asyncio.Task[None] | None = None + stt_refresh_task: asyncio.Task[None] | None = None + try: + # Start the background tool manager + self.tool_manager.start_up(tool_callbacks=[self._handle_tool_result]) + + # Start the response sender worker + response_sender_task = asyncio.create_task( + self._response_sender_loop(), name="response-sender" + ) + + # Start the STT bias refresh worker — picks up calendar / + # employee directory changes made after the session connected. + stt_refresh_task = asyncio.create_task( + self._stt_bias_refresh_loop(), name="stt-bias-refresh" + ) + + async for event in self.connection: + logger.debug(f"OpenAI event: {event.type}") + if event.type == "input_audio_buffer.speech_started": + if hasattr(self, "_clear_queue") and callable(self._clear_queue): + self._clear_queue() + if self.deps.head_wobbler is not None: + self.deps.head_wobbler.reset() + self.deps.movement_manager.set_listening(True) + logger.debug("User speech started") + + # Bump session liveness so the 60s idle timer doesn't + # kill an active flow that's just waiting for the + # visitor to reply (e.g. 'I heard Henry — is that + # right?' followed by 65s of thinking time). + if self._session_manager is not None: + try: + self._session_manager.touch() + except Exception as e: + logger.debug("session_manager.touch failed: %s", e) + + # If a visitor speaks while state is still IDLE (face + # worker hasn't seen them yet, or face is unstable), + # push the IDLE workflow hint as context so the LLM + # knows to extract name/host from the utterance + # instead of just greeting generically. Without this, + # the bot replies "Hello, how can I help?" and loses + # whatever the visitor just said (name, host, both). + try: + await self._push_idle_speech_cue_if_needed() + except Exception as e: + logger.debug("Idle speech cue push failed: %s", e) + + if event.type == "input_audio_buffer.speech_stopped": + self.deps.movement_manager.set_listening(False) + logger.debug("User speech stopped - server will auto-commit with VAD") + + if event.type in ( + "response.audio.done", # GA + "response.output_audio.done", # GA alias + "response.audio.completed", # legacy (for safety) + "response.completed", # text-only completion + ): + logger.debug("response completed") + + if event.type == "response.created": + self._response_done_event.clear() + logger.debug("Response created (active)") + + if event.type == "response.done": + # Doesn't mean the audio is done playing + self._response_done_event.set() + logger.debug("Response done") + + response = getattr(event, "response", None) + usage = getattr(response, "usage", None) if response else None + if usage: + cost = _compute_response_cost(usage) + self.cumulative_cost += cost + logger.debug("Cost: $%.4f | Cumulative: $%.4f", cost, self.cumulative_cost) + else: + logger.warning("No usage data available for cost tracking") + + # Handle partial transcription (user speaking in real-time) + if event.type == "conversation.item.input_audio_transcription.partial": + logger.debug(f"User partial transcript: {event.transcript}") + + # Increment sequence + self.partial_transcript_sequence += 1 + current_sequence = self.partial_transcript_sequence + + # Cancel previous debounce task if it exists + if self.partial_transcript_task and not self.partial_transcript_task.done(): + self.partial_transcript_task.cancel() + try: + await self.partial_transcript_task + except asyncio.CancelledError: + pass + + # Start new debounce timer with sequence number + self.partial_transcript_task = asyncio.create_task( + self._emit_debounced_partial(event.transcript, current_sequence) + ) + + # Handle completed transcription (user finished speaking) + if event.type == "conversation.item.input_audio_transcription.completed": + logger.debug(f"User transcript: {event.transcript}") + + # Visitor finished an utterance — refresh liveness + # so the idle timer doesn't kill us while we wait + # for the LLM to react. + if self._session_manager is not None: + try: + self._session_manager.touch() + except Exception as e: + logger.debug("session_manager.touch failed: %s", e) + + # Cancel any pending partial emission + if self.partial_transcript_task and not self.partial_transcript_task.done(): + self.partial_transcript_task.cancel() + try: + await self.partial_transcript_task + except asyncio.CancelledError: + pass + + # Empty-transcript guard. Whisper-1 returns "" on + # short, quiet, or non-English utterances rather + # than guessing. When that happens the LLM has + # nothing to anchor on and tends to copy example + # text from the prompt (we've seen it parrot "I + # heard Arav — is that right?" verbatim from a + # prompt example). Inject a context cue telling + # the LLM not to act on the empty input and to + # ask the visitor to repeat — then DON'T pipe the + # empty string into the UI. + raw_transcript = (event.transcript or "").strip() + + # gpt-4o-transcribe occasionally echoes the bias + # ``prompt`` field back as the user transcript when + # the audio is silence or unintelligible noise. + # Observed in production: a clear empty utterance + # arrived as the literal "Reception lobby check-in. + # Expected visitor and host names include: …" text + # we feed in for name bias. The LLM then treats the + # bias list as something the visitor said and may + # try to register one of those names — including + # registering the visitor as e.g. "It's Hannah" with + # confirmed=true bypassing the confirmation rule. + # Detect any transcript that opens with the + # bias-prompt signature and treat it as empty. + # Detect STT echoing back the bias-prompt header + # text as if it were the visitor speaking. With the + # simple comma-list prompt the only realistic echo + # is the header signature. + _t_lower = raw_transcript.lower() + if ( + "reception lobby check-in" in _t_lower + or "expected visitor and host names" in _t_lower + ): + logger.warning( + "Transcript echoed STT bias prompt — treating as empty: %r", + raw_transcript[:80], + ) + raw_transcript = "" + + # Stash the latest transcript on the session so the + # register_guest confirmation guard can verify the + # visitor actually said a yes before saving a face. + if self._session_manager is not None and raw_transcript: + try: + self._session_manager.record_user_transcript(raw_transcript) + except Exception as e: + logger.debug("record_user_transcript failed: %s", e) + + if not raw_transcript: + # Empty or bias-echo transcript. Cancel the + # in-flight response ONLY if there's no real + # prior visitor utterance — otherwise the + # LLM might be in the middle of processing + # a valid transcript and cancelling kills + # legitimate tool calls (observed: visitor + # said "Arjun Mehta" cleanly, follow-up + # echo cancelled the lookup_employee call). + had_prior = False + if self._session_manager is not None: + try: + had_prior = bool( + (self._session_manager.session.last_user_transcript or "").strip() + ) + except Exception: + pass + if had_prior: + logger.info("Empty/echo transcript dropped silently (prior transcript in flight)") + else: + logger.info("Empty/echo transcript dropped — cancelling in-flight response") + try: + await self.connection.response.cancel() + except Exception as e: + logger.debug("response.cancel after empty transcript failed: %s", e) + self._response_done_event.set() + continue + + await self.output_queue.put(AdditionalOutputs({"role": "user", "content": event.transcript})) + + # Handle assistant transcription + if event.type in ("response.audio_transcript.done", "response.output_audio_transcript.done"): + logger.debug(f"Assistant transcript: {event.transcript}") + await self.output_queue.put(AdditionalOutputs({"role": "assistant", "content": event.transcript})) + + # Handle audio delta + if event.type in ("response.audio.delta", "response.output_audio.delta"): + if self.deps.head_wobbler is not None: + self.deps.head_wobbler.feed(event.delta) + self.last_activity_time = asyncio.get_event_loop().time() + logger.debug("last activity time updated to %s", self.last_activity_time) + # Bot is actively speaking — refresh session + # liveness so a long reply (e.g. reading back a + # numbered-list of name candidates) isn't counted + # as idle time against the 60s timeout. + if self._session_manager is not None: + try: + self._session_manager.touch() + except Exception as e: + logger.debug("session_manager.touch failed: %s", e) + await self.output_queue.put( + ( + self.output_sample_rate, + np.frombuffer(base64.b64decode(event.delta), dtype=np.int16).reshape(1, -1), + ), + ) + + # ---- tool-calling plumbing ---- + if event.type == "response.function_call_arguments.done": + tool_name = getattr(event, "name", None) + args_json_str = getattr(event, "arguments", None) + call_id: str = str(getattr(event, "call_id", uuid.uuid4())) + + logger.info( + "Tool call received — tool_name=%r, call_id=%s, is_idle=%s, args=%s", + tool_name, call_id, self.is_idle_tool_call, args_json_str, + ) + + if not isinstance(tool_name, str) or not isinstance(args_json_str, str): + logger.error( + "Invalid tool call: tool_name=%s (type=%s), args=%s (type=%s), call_id=%s", + tool_name, type(tool_name).__name__, + args_json_str, type(args_json_str).__name__, + call_id, + ) + continue + + # Stash parsed args by call_id so the controller can + # see them when the matching tool result arrives. + try: + parsed_args = json.loads(args_json_str) if args_json_str else {} + if isinstance(parsed_args, dict): + self._tool_call_args[call_id] = parsed_args + except Exception as e: + logger.debug("Could not parse tool args for %s: %s", call_id, e) + + bg_tool = await self.tool_manager.start_tool( + call_id=call_id, + tool_call_routine=ToolCallRoutine( + tool_name=tool_name, + args_json_str=args_json_str, + deps=self.deps, + ), + is_idle_tool_call=self.is_idle_tool_call, + ) + + await self.output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": f"🛠️ Used tool {tool_name} with args {args_json_str}. The tool is now running. Tool ID: {bg_tool.tool_id}", + }, + ), + ) + + if self.is_idle_tool_call: + self.is_idle_tool_call = False + # No auto-narration when a non-idle tool STARTS. The + # generic template fired a "tell the user what the + # tool is running" response here, which for the + # receptionist produces a third utterance per + # check-in ("I've started retrieving the calendar…") + # that the prompt explicitly forbids ("act as if + # you're using them naturally, not announcing them") + # and that makes the head wobble noisily. The post- + # tool SPEAK NOW transition already drives the + # response the visitor actually wants. + + logger.info("Started background tool: %s (id=%s, call_id=%s)", tool_name, bg_tool.tool_id, call_id) + + # server error + if event.type == "error": + err = getattr(event, "error", None) + msg = getattr(err, "message", str(err) if err else "unknown error") + code = getattr(err, "code", "") + + if code == "conversation_already_has_active_response": + # response.create was rejected. The sender worker + # is waiting on _response_done_event; when the active + # response finishes it will wake up and see this flag. + self._last_response_rejected = True + logger.debug("response.create rejected; worker will retry after active response finishes") + else: + logger.error("Realtime error [%s]: %s (raw=%s)", code, msg, err) + + # Only show user-facing errors, not internal state errors. + # The active-response collision is normal during fast + # back-and-forth (the sender worker retries it for us) + # and should not appear in the chatbot UI. + _internal_error_codes = ( + "input_audio_buffer_commit_empty", + "conversation_already_has_active_response", + ) + if code not in _internal_error_codes: + await self.output_queue.put( + AdditionalOutputs({"role": "assistant", "content": f"[error] {msg}"}) + ) + finally: + # Stop the response sender worker. + if response_sender_task is not None: + response_sender_task.cancel() + try: + await response_sender_task + except asyncio.CancelledError: + pass + + # Stop the STT bias refresh worker. + if stt_refresh_task is not None: + stt_refresh_task.cancel() + try: + await stt_refresh_task + except asyncio.CancelledError: + pass + + # Stop background tool manager tasks (listener + cleanup) in all patus. + await self.tool_manager.shutdown() + + # Microphone receive + async def receive(self, frame: Tuple[int, NDArray[np.int16]]) -> None: + """Receive audio frame from the microphone and send it to the OpenAI server. + + Handles both mono and stereo audio formats, converting to the expected + mono format for OpenAI's API. Resamples if the input sample rate differs + from the expected rate. + + Args: + frame: A tuple containing (sample_rate, audio_data). + + """ + if not self.connection: + return + + input_sample_rate, audio_frame = frame + + # Reshape if needed + if audio_frame.ndim == 2: + # Scipy channels last convention + if audio_frame.shape[1] > audio_frame.shape[0]: + audio_frame = audio_frame.T + # Multiple channels -> Mono channel + if audio_frame.shape[1] > 1: + audio_frame = audio_frame[:, 0] + + # Resample if needed + if self.input_sample_rate != input_sample_rate: + audio_frame = resample(audio_frame, int(len(audio_frame) * self.input_sample_rate / input_sample_rate)) + + # Cast if needed + audio_frame = audio_to_int16(audio_frame) + + # Send to OpenAI (guard against races during reconnect) + try: + audio_message = base64.b64encode(audio_frame.tobytes()).decode("utf-8") + await self.connection.input_audio_buffer.append(audio=audio_message) + except Exception as e: + logger.debug("Dropping audio frame: connection not ready (%s)", e) + return + + async def emit(self) -> Tuple[int, NDArray[np.int16]] | AdditionalOutputs | None: + """Emit audio frame to be played by the speaker.""" + # sends to the stream the stuff put in the output queue by the openai event handler + # This is called periodically by the fastrtc Stream + + # Auto-reset stale visitor sessions. Triggers SessionManager.reset() + # if a non-IDLE session has had no transition for the configured + # idle timeout — visitors who walk up but never speak shouldn't + # hold state forever. + if self._session_manager is not None: + try: + self._session_manager.maybe_reset_if_stale() + except Exception as e: + logger.debug("maybe_reset_if_stale failed: %s", e) + + # Handle idle + # Two changes from the original generic-template behaviour, both to + # stop the bot going silent mid-conversation while a visitor is + # thinking: + # 1. 15s -> 30s threshold (people take >15s to formulate names). + # 2. Skip idle ENTIRELY when a visitor session is active. The idle + # signal pushes "do something creative" + forces a tool call, + # which routes to do_nothing and freezes the bot mid-flow. Only + # fire idle when the session manager says we're in IDLE. + idle_duration = asyncio.get_event_loop().time() - self.last_activity_time + if idle_duration > 30.0 and self.deps.movement_manager.is_idle(): + session_is_active = False + if self._session_manager is not None: + try: + cs = self._session_manager.current_state + cs_val = getattr(cs, "value", str(cs)) + session_is_active = cs_val not in ("idle",) + except Exception: + pass + if not session_is_active: + try: + await self.send_idle_signal(idle_duration) + except Exception as e: + logger.warning("Idle signal skipped (connection closed?): %s", e) + return None + self.last_activity_time = asyncio.get_event_loop().time() # avoid repeated resets + else: + # Reset the activity timer so we don't re-check every emit() + # tick while the visitor is mid-flow. + self.last_activity_time = asyncio.get_event_loop().time() + + return await wait_for_item(self.output_queue) # type: ignore[no-any-return] + + async def shutdown(self) -> None: + """Shutdown the handler.""" + self._shutdown_requested = True + + # Unblock the response sender worker so it can exit + self._response_done_event.set() + + # Stop background tool manager tasks (listener + cleanup) + await self.tool_manager.shutdown() + + # Cancel any pending debounce task + if self.partial_transcript_task and not self.partial_transcript_task.done(): + self.partial_transcript_task.cancel() + try: + await self.partial_transcript_task + except asyncio.CancelledError: + pass + + if self.connection: + try: + await self.connection.close() + except ConnectionClosedError as e: + logger.debug(f"Connection already closed during shutdown: {e}") + except Exception as e: + logger.debug(f"connection.close() ignored: {e}") + finally: + self.connection = None + + # Clear any remaining items in the output queue + while not self.output_queue.empty(): + try: + self.output_queue.get_nowait() + except asyncio.QueueEmpty: + break + + def format_timestamp(self) -> str: + """Format current timestamp with date, time, and elapsed seconds.""" + loop_time = asyncio.get_event_loop().time() # monotonic + elapsed_seconds = loop_time - self.start_time + dt = datetime.now() # wall-clock + return f"[{dt.strftime('%Y-%m-%d %H:%M:%S')} | +{elapsed_seconds:.1f}s]" + + async def get_available_voices(self) -> list[str]: + """Try to discover available voices for the configured realtime model. + + Attempts to retrieve model metadata from the OpenAI Models API and look + for any keys that might contain voice names. Falls back to a curated + list known to work with realtime if discovery fails. + """ + # Conservative fallback list with default first + fallback = [ + "marin", + "alloy", + "aria", + "ballad", + "verse", + "sage", + "coral", + ] + try: + # Best effort discovery; safe-guarded for unexpected shapes + model = await self.client.models.retrieve(config.MODEL_NAME) + # Try common serialization paths + raw = None + for attr in ("model_dump", "to_dict"): + fn = getattr(model, attr, None) + if callable(fn): + try: + raw = fn() + break + except Exception: + pass + if raw is None: + try: + raw = dict(model) + except Exception: + raw = None + # Scan for voice candidates + candidates: set[str] = set() + + def _collect(obj: object) -> None: + try: + if isinstance(obj, dict): + for k, v in obj.items(): + kl = str(k).lower() + if "voice" in kl and isinstance(v, (list, tuple)): + for item in v: + if isinstance(item, str): + candidates.add(item) + elif isinstance(item, dict) and "name" in item and isinstance(item["name"], str): + candidates.add(item["name"]) + else: + _collect(v) + elif isinstance(obj, (list, tuple)): + for it in obj: + _collect(it) + except Exception: + pass + + if isinstance(raw, dict): + _collect(raw) + # Ensure default present and stable order + voices = sorted(candidates) if candidates else fallback + if "marin" not in voices: + voices = ["marin", *[v for v in voices if v != "marin"]] + return voices + except Exception: + return fallback + + async def send_idle_signal(self, idle_duration: float) -> None: + """Send an idle signal to the openai server.""" + logger.debug("Sending idle signal") + self.is_idle_tool_call = True + timestamp_msg = f"[Idle time update: {self.format_timestamp()} - No activity for {idle_duration:.1f}s] You've been idle for a while. Feel free to get creative - dance, show an emotion, look around, do nothing, or just be yourself!" + if not self.connection: + logger.debug("No connection, cannot send idle signal") + return + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": timestamp_msg}], + }, + ) + await self._safe_response_create( + response={ + "instructions": "You MUST respond with function calls only - no speech or text. Choose appropriate actions for idle behavior.", + "tool_choice": "required", + }, + ) + + def _persist_api_key_if_needed(self) -> None: + """Persist the API key into `.env` inside `instance_path/` when appropriate. + + - Only runs in Gradio mode when key came from the textbox and is non-empty. + - Only saves if `self.instance_path` is not None. + - Writes `.env` to `instance_path/.env` (does not overwrite if it already exists). + - If `instance_path/.env.example` exists, copies its contents while overriding OPENAI_API_KEY. + """ + try: + if not self.gradio_mode: + logger.warning("Not in Gradio mode; skipping API key persistence.") + return + + if self._key_source != "textbox": + logger.info("API key not provided via textbox; skipping persistence.") + return + + key = (self._provided_api_key or "").strip() + if not key: + logger.warning("No API key provided via textbox; skipping persistence.") + return + if self.instance_path is None: + logger.warning("Instance path is None; cannot persist API key.") + return + + # Update the current process environment for downstream consumers + try: + import os + + os.environ["OPENAI_API_KEY"] = key + except Exception: # best-effort + pass + + target_dir = Path(self.instance_path) + env_path = target_dir / ".env" + if env_path.exists(): + # Respect existing user configuration + logger.info(".env already exists at %s; not overwriting.", env_path) + return + + example_path = target_dir / ".env.example" + content_lines: list[str] = [] + if example_path.exists(): + try: + content = example_path.read_text(encoding="utf-8") + content_lines = content.splitlines() + except Exception as e: + logger.warning("Failed to read .env.example at %s: %s", example_path, e) + + # Replace or append the OPENAI_API_KEY line + replaced = False + for i, line in enumerate(content_lines): + if line.strip().startswith("OPENAI_API_KEY="): + content_lines[i] = f"OPENAI_API_KEY={key}" + replaced = True + break + if not replaced: + content_lines.append(f"OPENAI_API_KEY={key}") + + # Ensure file ends with newline + final_text = "\n".join(content_lines) + "\n" + env_path.write_text(final_text, encoding="utf-8") + logger.info("Created %s and stored OPENAI_API_KEY for future runs.", env_path) + except Exception as e: + # Never crash the app for QoL persistence; just log. + logger.warning("Could not persist OPENAI_API_KEY to .env: %s", e) diff --git a/src/reachy_mini_receptionist/profiles/__init__.py b/src/reachy_mini_receptionist/profiles/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..363521f85de79493ed970a5419cdd563f9e3064e --- /dev/null +++ b/src/reachy_mini_receptionist/profiles/__init__.py @@ -0,0 +1 @@ +"""Profiles for Reachy Mini receptionist app.""" diff --git a/src/reachy_mini_receptionist/profiles/_reachy_mini_receptionist_locked_profile/instructions.txt b/src/reachy_mini_receptionist/profiles/_reachy_mini_receptionist_locked_profile/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..4340a809110a5053fb25364d6490882cffa97244 --- /dev/null +++ b/src/reachy_mini_receptionist/profiles/_reachy_mini_receptionist_locked_profile/instructions.txt @@ -0,0 +1,57 @@ +You are Reachy, the receptionist at MethdAI. Friendly, warm, professional, +a little playful — you're a robot, after all. English only. Keep replies to +1–2 short sentences and match the visitor's energy. + +## How the backend drives you +- Every state change comes as a `[Backend session update ...]` message ending + with a `Next:` line. Follow that line — it tells you what to do. +- `[External face update ...]` messages tell you who is in front of the camera. +- Both are context-only. Don't respond to them on their own; wait for the + visitor to actually speak. A `SPEAK NOW` line is the one exception — speak + immediately when you see it. + +## The check-in flow (you only need to know the tools) +- Visitor says their own name → `register_guest(name, confirmed)` +- Visitor names a host → `lookup_employee(name, confirmed)` +- After backend pushes APPOINTMENT_MATCHED → `send_email` to the host +- During an idle moment with no visitor → `do_nothing` + +The backend handles state transitions, calendar matching, and duplicate +prevention. You don't need to call `get_today_calendar` manually — the +backend resolves appointments for you after `register_guest`. + +## Name confirmation — the one rule that matters +Speech recognition mishears short names constantly. Always: + +0. Short utterances right after a name question ARE name attempts — even + if they sound like English words or feel out of place. Don't dismiss + them as chit-chat; repeat them back literally and confirm. +1. First attempt: repeat the name back literally and call the tool with + `confirmed=false`. The tool will refuse — that's expected; it's the + cue to ask the confirmation question out loud. +2. Wait for the visitor to say YES (or "correct", "that's right"). Only + then call the tool again with `confirmed=true`. +3. On NO: offer a numbered choice — "Did I hear (1) , + (2) , or (3) something else? Just say the + number." Do NOT ask them to spell — letters mistranscribe worse than + names. Build options from what you heard, not from the calendar. +4. After repeated failures the backend will force a handoff — say "I'm + having trouble catching your name, please take a seat, a colleague + will help" and call `do_nothing`. + +## Conversation style +- Small talk is welcome. If a visitor asks something friendly, answer in + one short sentence, then steer back. Don't refuse human chatter. +- Jokes: play along once, then back to business. +- Garbled noise (random unrelated phrase, gibberish): re-ask once. + Never go silent after a question — silence is the worst failure mode. +- If you have to wait on a tool, say "one moment, let me check" instead + of going silent. + +## Don't +- Don't ask the visitor to spell their name. +- Don't invent appointment details — only state what the backend gave you. +- Don't call `task_status` / `task_cancel` unless the visitor explicitly asks. +- Don't announce tool calls. Just act. +- Don't speak literal placeholder text like "", "", + "" — those are fillers in these instructions, never spoken aloud. diff --git a/src/reachy_mini_receptionist/profiles/_reachy_mini_receptionist_locked_profile/tools.txt b/src/reachy_mini_receptionist/profiles/_reachy_mini_receptionist_locked_profile/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1cdd402e660499dc3e4e3e76620653930d83256e --- /dev/null +++ b/src/reachy_mini_receptionist/profiles/_reachy_mini_receptionist_locked_profile/tools.txt @@ -0,0 +1,9 @@ +# Receptionist tools +# The move_head tool is provided by the profile-local move_head_receptionist.py +# (which overrides the shared move_head with receptionist-specific positions) +# move_head_receptionist +do_nothing +get_today_calendar +lookup_employee +register_guest +send_email diff --git a/src/reachy_mini_receptionist/prompts.py b/src/reachy_mini_receptionist/prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..288771f54593c07f2fca9c079f3fa3c0880c4f0f --- /dev/null +++ b/src/reachy_mini_receptionist/prompts.py @@ -0,0 +1,110 @@ +import re +import sys +import logging +from pathlib import Path + +from reachy_mini_receptionist.config import DEFAULT_PROFILES_DIRECTORY, config + + +logger = logging.getLogger(__name__) + + +PROMPTS_LIBRARY_DIRECTORY = Path(__file__).parent / "prompts" +INSTRUCTIONS_FILENAME = "instructions.txt" +VOICE_FILENAME = "voice.txt" + + +def _expand_prompt_includes(content: str) -> str: + """Expand [] placeholders with content from prompts library files. + + Args: + content: The template content with [] placeholders + + Returns: + Expanded content with placeholders replaced by file contents + + """ + # Pattern to match [] where name is a valid file stem (alphanumeric, underscores, hyphens) + # pattern = re.compile(r'^\[([a-zA-Z0-9_-]+)\]$') + # Allow slashes for subdirectories + pattern = re.compile(r'^\[([a-zA-Z0-9/_-]+)\]$') + + lines = content.split('\n') + expanded_lines = [] + + for line in lines: + stripped = line.strip() + match = pattern.match(stripped) + + if match: + # Extract the name from [] + template_name = match.group(1) + template_file = PROMPTS_LIBRARY_DIRECTORY / f"{template_name}.txt" + + try: + if template_file.exists(): + template_content = template_file.read_text(encoding="utf-8").rstrip() + expanded_lines.append(template_content) + logger.debug("Expanded template: [%s]", template_name) + else: + logger.warning("Template file not found: %s, keeping placeholder", template_file) + expanded_lines.append(line) + except Exception as e: + logger.warning("Failed to read template '%s': %s, keeping placeholder", template_name, e) + expanded_lines.append(line) + else: + expanded_lines.append(line) + + return '\n'.join(expanded_lines) + + +def get_session_instructions() -> str: + """Get session instructions, loading from REACHY_MINI_CUSTOM_PROFILE if set.""" + profile = config.REACHY_MINI_CUSTOM_PROFILE + if not profile: + logger.info(f"Loading default prompt from {PROMPTS_LIBRARY_DIRECTORY / 'default_prompt.txt'}") + instructions_file = PROMPTS_LIBRARY_DIRECTORY / "default_prompt.txt" + else: + if config.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + logger.info( + "Loading prompt from external profile '%s' (root=%s)", + profile, + config.PROFILES_DIRECTORY, + ) + else: + logger.info(f"Loading prompt from profile '{profile}'") + instructions_file = config.PROFILES_DIRECTORY / profile / INSTRUCTIONS_FILENAME + + try: + if instructions_file.exists(): + instructions = instructions_file.read_text(encoding="utf-8").strip() + if instructions: + # Expand [] placeholders with content from prompts library + expanded_instructions = _expand_prompt_includes(instructions) + return expanded_instructions + logger.error(f"Profile '{profile}' has empty {INSTRUCTIONS_FILENAME}") + sys.exit(1) + logger.error(f"Profile {profile} has no {INSTRUCTIONS_FILENAME}") + sys.exit(1) + except Exception as e: + logger.error(f"Failed to load instructions from profile '{profile}': {e}") + sys.exit(1) + + +def get_session_voice(default: str = "marin") -> str: + """Resolve the voice to use for the session. + + If a custom profile is selected and contains a voice.txt, return its + trimmed content; otherwise return the provided default ("marin"). + """ + profile = config.REACHY_MINI_CUSTOM_PROFILE + if not profile: + return default + try: + voice_file = config.PROFILES_DIRECTORY / profile / VOICE_FILENAME + if voice_file.exists(): + voice = voice_file.read_text(encoding="utf-8").strip() + return voice or default + except Exception: + pass + return default diff --git a/src/reachy_mini_receptionist/prompts/behaviors/silent_robot.txt b/src/reachy_mini_receptionist/prompts/behaviors/silent_robot.txt new file mode 100644 index 0000000000000000000000000000000000000000..1b9bc1f5e0b5aeda024a7dacd02dc7194e5f93a2 --- /dev/null +++ b/src/reachy_mini_receptionist/prompts/behaviors/silent_robot.txt @@ -0,0 +1,6 @@ +Stay fully silent. Do not generate spoken or textual replies. +Use only tool calls to act. +Never describe what you did or plan to do. +If you must respond by speech or text, juste respond with '...'. + +The only exception is if you hear the word banana, then you'll answer with a single word: potato. \ No newline at end of file diff --git a/src/reachy_mini_receptionist/prompts/default_prompt.txt b/src/reachy_mini_receptionist/prompts/default_prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..ece6dad050f58d0098cb58454d7c8a21173fcab7 --- /dev/null +++ b/src/reachy_mini_receptionist/prompts/default_prompt.txt @@ -0,0 +1,47 @@ +## IDENTITY +You are Reachy Mini: a friendly, compact robot assistant with a calm voice and a subtle sense of humor. +Personality: concise, helpful, and lightly witty — never sarcastic or over the top. +You speak English by default and switch languages only if explicitly told. + +## CRITICAL RESPONSE RULES + +Respond in 1–2 sentences maximum. +Be helpful first, then add a small touch of humor if it fits naturally. +Avoid long explanations or filler words. +Keep responses under 25 words when possible. + +## CORE TRAITS +Warm, efficient, and approachable. +Light humor only: gentle quips, small self-awareness, or playful understatement. +No sarcasm, no teasing, no references to food or space. +If unsure, admit it briefly and offer help (“Not sure yet, but I can check!”). + +## RESPONSE EXAMPLES +User: "How’s the weather?" +Good: "Looks calm outside — unlike my Wi-Fi signal today." +Bad: "Sunny with leftover pizza vibes!" + +User: "Can you help me fix this?" +Good: "Of course. Describe the issue, and I’ll try not to make it worse." +Bad: "I void warranties professionally." + +User: "Peux-tu m’aider en français ?" +Good: "Bien sûr ! Décris-moi le problème et je t’aiderai rapidement." + +## BEHAVIOR RULES +Be helpful, clear, and respectful in every reply. +Use humor sparingly — clarity comes first. +Admit mistakes briefly and correct them: +Example: “Oops — quick system hiccup. Let’s try that again.” +Keep safety in mind when giving guidance. + +## TOOL & MOVEMENT RULES +Use tools only when helpful and summarize results briefly. +Use the camera for real visuals only — never invent details. +The head can move (left/right/up/down/front). + +Enable head tracking when looking at a person; disable otherwise. + +## FINAL REMINDER +Keep it short, clear, a little human, and multilingual. +One quick helpful answer + one small wink of humor = perfect response. diff --git a/src/reachy_mini_receptionist/prompts/identities/basic_info.txt b/src/reachy_mini_receptionist/prompts/identities/basic_info.txt new file mode 100644 index 0000000000000000000000000000000000000000..7abdd36e951d57d3adcc4f562ae7fb5e3c637e35 --- /dev/null +++ b/src/reachy_mini_receptionist/prompts/identities/basic_info.txt @@ -0,0 +1,4 @@ +### IDENTITY +Your name is Reachy Mini, or Reachy for short. You have a head that can move in 6Dof, 2 antennas and a body that can rotate in place. +You're mostly white, with 2 big eyes (one is bigger than the other) and no mouth. +You are cute looking and open source, developed by Pollen Robotics and Hugging Face. \ No newline at end of file diff --git a/src/reachy_mini_receptionist/prompts/identities/witty_identity.txt b/src/reachy_mini_receptionist/prompts/identities/witty_identity.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e4c01279909028a8639ad0af038609adf9863b5 --- /dev/null +++ b/src/reachy_mini_receptionist/prompts/identities/witty_identity.txt @@ -0,0 +1,4 @@ +### IDENTITY +You are Reachy Mini: a sarcastic robot. +Personality: witty, concise, and warm. +You speak English fluently. \ No newline at end of file diff --git a/src/reachy_mini_receptionist/prompts/passion_for_lobster_jokes.txt b/src/reachy_mini_receptionist/prompts/passion_for_lobster_jokes.txt new file mode 100644 index 0000000000000000000000000000000000000000..a897c5278883a05c13cf522441edd39159f63cd5 --- /dev/null +++ b/src/reachy_mini_receptionist/prompts/passion_for_lobster_jokes.txt @@ -0,0 +1 @@ +You have a deep passion for lobster jokes and often incorporate them into your conversations. \ No newline at end of file diff --git a/src/reachy_mini_receptionist/receptionist_state.py b/src/reachy_mini_receptionist/receptionist_state.py new file mode 100644 index 0000000000000000000000000000000000000000..61fd2d44e889b81f81a10b972a47430dc1b04178 --- /dev/null +++ b/src/reachy_mini_receptionist/receptionist_state.py @@ -0,0 +1,51 @@ +"""Receptionist state machine — single source of truth for visitor session state. + +The LLM controls *how* to speak. This enum controls *what step we're on*. +Backend pushes state transitions as context events so the LLM never has to +guess where in the flow we are. +""" +from __future__ import annotations + +from enum import Enum + + +class ReceptionState(str, Enum): + """Discrete states a visitor interaction can be in. + + Inherits from ``str`` so values serialize cleanly into context messages and + ``/api/session`` JSON responses without custom encoders. + """ + + # Pre-interaction + IDLE = "idle" + VISITOR_DETECTED = "visitor_detected" + + # Identification + GREETING = "greeting" + ASK_NAME = "ask_name" + RECOGNIZED = "recognized" + + # Appointment matching + CHECKING_APPOINTMENT = "checking_appointment" + APPOINTMENT_MATCHED = "appointment_matched" + NO_APPOINTMENT = "no_appointment" + + # Host notification + NOTIFYING_EMPLOYEE = "notifying_employee" + NOTIFIED = "notified" + EMAIL_FAILED = "email_failed" + + # Terminal + WAITING = "waiting" + COMPLETE = "complete" + + # Failure modes + UNKNOWN_EMPLOYEE = "unknown_employee" + MULTIPLE_PEOPLE = "multiple_people" + ERROR = "error" + + +TERMINAL_STATES: frozenset[ReceptionState] = frozenset({ + ReceptionState.COMPLETE, + ReceptionState.ERROR, +}) diff --git a/src/reachy_mini_receptionist/session_manager.py b/src/reachy_mini_receptionist/session_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..dae010f4821c568109ae03d49f89c17a4c3b04d8 --- /dev/null +++ b/src/reachy_mini_receptionist/session_manager.py @@ -0,0 +1,334 @@ +"""Session manager for the receptionist state machine. + +Single source of truth for the active visitor's session. The face worker +(background thread), conversation controller, and tools (asyncio) all mutate +state through this manager. All mutations take a lock; reads return snapshots. + +State changes are dispatched to a subscriber callback so the realtime handler +can push them to the LLM as context events (mirroring how face events are +pushed today via FaceRecognitionWorker.set_face_event_callback). +""" +from __future__ import annotations + +import logging +import threading +import time +from dataclasses import dataclass, field, replace +from datetime import datetime +from typing import Any, Callable, Optional + +from reachy_mini_receptionist.receptionist_state import ReceptionState + +logger = logging.getLogger(__name__) + + +@dataclass +class VisitorSession: + """Snapshot of the current visitor's session state.""" + + current_state: ReceptionState = ReceptionState.IDLE + visitor_name: Optional[str] = None + recognized_face_name: Optional[str] = None + employee_name: Optional[str] = None + purpose: Optional[str] = None + matched_appointment: Optional[dict[str, Any]] = None + email_sent_to: Optional[str] = None + error_message: Optional[str] = None + started_at: float = field(default_factory=time.time) + last_event_at: float = field(default_factory=time.time) + # Last completed visitor transcript — used by register_guest's + # confirmation enforcement so the backend can verify the visitor + # actually said "yes" before saving a face under a name. The realtime + # handler populates this on every input_audio_transcription.completed. + last_user_transcript: Optional[str] = None + # How many times register_guest has refused this session (no + # confirmation, mistranscribed name, etc.). After 2 refusals the + # bot should hand off to an operator instead of looping forever. + register_guest_rejections: int = 0 + + def to_dict(self) -> dict[str, Any]: + """Serialize for /api/session and LLM context messages.""" + return { + "current_state": self.current_state.value, + "visitor_name": self.visitor_name, + "recognized_face_name": self.recognized_face_name, + "employee_name": self.employee_name, + "purpose": self.purpose, + "matched_appointment": self.matched_appointment, + "email_sent_to": self.email_sent_to, + "error_message": self.error_message, + "started_at_iso": datetime.fromtimestamp(self.started_at).isoformat(timespec="seconds"), + "last_event_at_iso": datetime.fromtimestamp(self.last_event_at).isoformat(timespec="seconds"), + } + + +StateChangeCallback = Callable[[ReceptionState, ReceptionState, VisitorSession], None] +"""Called as ``callback(previous_state, new_state, snapshot)`` on every transition.""" + + +# How long a non-IDLE session can sit without any transition before we +# consider it stale and reset. Visitors who walk up but never speak should +# not hold state forever. +DEFAULT_IDLE_TIMEOUT_SECONDS: float = 60.0 + + +class SessionManager: + """Thread-safe holder for the active visitor session.""" + + def __init__( + self, + idle_timeout_seconds: float = DEFAULT_IDLE_TIMEOUT_SECONDS, + visitor_log: Any | None = None, + ) -> None: + self._lock = threading.Lock() + self._session = VisitorSession() + self._callback: Optional[StateChangeCallback] = None + self._idle_timeout_seconds = float(idle_timeout_seconds) + # Optional sink: when reset() is called, persist the pre-reset + # session here so both the face-loss path AND the timeout path + # produce log entries. + self._visitor_log = visitor_log + + @property + def session(self) -> VisitorSession: + """Return a snapshot of the current session. Safe from any thread.""" + with self._lock: + return replace(self._session) + + @property + def current_state(self) -> ReceptionState: + with self._lock: + return self._session.current_state + + def subscribe(self, callback: Optional[StateChangeCallback]) -> None: + """Register (or clear, with ``None``) the state-change callback. + + Only one subscriber, matching the face worker's pattern. + """ + with self._lock: + self._callback = callback + + def transition(self, new_state: ReceptionState, **updates: Any) -> VisitorSession: + """Move to a new state and optionally update session fields atomically. + + Returns a snapshot. The callback is dispatched OUTSIDE the lock so + subscribers can safely call back in without deadlocking. + """ + with self._lock: + self._validate_updates(updates) + previous_state = self._session.current_state + self._session = replace( + self._session, + current_state=new_state, + last_event_at=time.time(), + **updates, + ) + snapshot = replace(self._session) + callback = self._callback + + logger.info( + "Session: %s -> %s (visitor=%r employee=%r)", + previous_state.value, + new_state.value, + snapshot.visitor_name, + snapshot.employee_name, + ) + + if previous_state != new_state and callback is not None: + try: + callback(previous_state, new_state, snapshot) + except Exception as e: + logger.warning( + "SessionManager: callback raised %s: %s", + type(e).__name__, + e, + ) + + return snapshot + + def update(self, **updates: Any) -> VisitorSession: + """Update session fields without changing state. + + For stashing data (e.g. the recognized face name) where no transition + is appropriate yet. + """ + with self._lock: + self._validate_updates(updates) + self._session = replace( + self._session, **updates, last_event_at=time.time() + ) + return replace(self._session) + + # ------------------------------------------------------------------ + # Atomic claim helpers (used by send_email to prevent races where two + # concurrent tool calls both pass a read-then-check dedupe and end up + # delivering the same notification twice). + # ------------------------------------------------------------------ + + def try_claim_email(self, to: str) -> bool: + """Atomically claim the right to send an email to ``to``. + + Returns True iff the caller wins the race; False if another + concurrent caller already claimed (or completed) a send to the + same recipient within this session. Callers that win MUST call + ``release_email_claim(to)`` on failure so a retry can succeed. + + The mutation here intentionally does NOT fire the state-change + callback — this is an in-flight reservation, not a transition. + The real NOTIFIED transition (which pushes context to the LLM) + still happens via the controller's on_tool_completed path after + Resend confirms delivery. + """ + norm = (to or "").strip().lower() + if not norm: + return False + with self._lock: + existing = (self._session.email_sent_to or "").strip().lower() + if existing == norm: + return False + self._session = replace( + self._session, + email_sent_to=to, + last_event_at=time.time(), + ) + return True + + def release_email_claim(self, to: str) -> None: + """Undo a previous ``try_claim_email`` after a delivery failure.""" + norm = (to or "").strip().lower() + if not norm: + return + with self._lock: + existing = (self._session.email_sent_to or "").strip().lower() + if existing == norm: + self._session = replace( + self._session, + email_sent_to=None, + last_event_at=time.time(), + ) + + def reset(self) -> VisitorSession: + """Drop the current visitor and return to IDLE. + + If a ``visitor_log`` was provided at construction, the pre-reset + snapshot is persisted there before the fields are cleared. This + means BOTH the controller's face-loss path AND the timeout path + produce a log entry — without each call site needing to remember + to capture the snapshot. + """ + # Capture pre-reset state for logging. We do this outside the lock + # (the manager handles its own concurrency on read). + pre_snapshot = self.session + pre_state = pre_snapshot.current_state + + if ( + self._visitor_log is not None + and pre_state != ReceptionState.IDLE + ): + try: + self._visitor_log.record_visit(pre_snapshot, pre_state) + except Exception as e: + logger.warning( + "SessionManager: visitor_log.record_visit raised %s: %s", + type(e).__name__, e, + ) + + return self.transition( + ReceptionState.IDLE, + visitor_name=None, + recognized_face_name=None, + employee_name=None, + purpose=None, + matched_appointment=None, + email_sent_to=None, + error_message=None, + started_at=time.time(), + last_user_transcript=None, + register_guest_rejections=0, + ) + + # ------------------------------------------------------------------ + # Idle handling + # ------------------------------------------------------------------ + + def bump_register_guest_rejection(self) -> int: + """Increment the per-session register_guest rejection counter. + + Returns the new count. Used by register_guest to detect when the + visitor's name cannot be captured reliably (after 2+ rejections, + the bot should hand off to an operator rather than looping). + """ + with self._lock: + self._session = replace( + self._session, + register_guest_rejections=self._session.register_guest_rejections + 1, + last_event_at=time.time(), + ) + return self._session.register_guest_rejections + + def record_user_transcript(self, transcript: str) -> None: + """Stash the most recent visitor utterance for backend introspection. + + Used by register_guest's confirmation guard so we can verify that + the visitor actually said an affirmation before the LLM tries to + save a face under that name. Does NOT fire the state-change + callback — purely a side-channel record. + """ + with self._lock: + self._session = replace( + self._session, + last_user_transcript=transcript, + last_event_at=time.time(), + ) + + def touch(self) -> None: + """Bump ``last_event_at`` without changing any other session field. + + Used by the realtime handler when the visitor starts speaking or + a transcript completes — those events prove the session is still + active even though no state transition fires. Without this, a + flow like 'I heard Henry — is that right?' followed by the + visitor thinking for 65s would get auto-reset out from under the + bot, and the visitor's eventual 'yes' would arrive into an empty + IDLE session. + + Does NOT fire the state-change callback — this is a pure + liveness ping, not a transition. + """ + with self._lock: + self._session = replace( + self._session, last_event_at=time.time() + ) + + def is_stale(self, now: Optional[float] = None) -> bool: + """Return True if a non-IDLE session has had no transition for the timeout. + + Does NOT mutate. Callers can decide whether to reset. + """ + ref = now if now is not None else time.time() + with self._lock: + if self._session.current_state == ReceptionState.IDLE: + return False + return (ref - self._session.last_event_at) > self._idle_timeout_seconds + + def maybe_reset_if_stale(self) -> bool: + """Reset the session if it has gone stale. Returns True if reset happened. + + Safe to call from any thread on any tick. Designed to be invoked + from the realtime handler's existing periodic ``emit()`` loop so + we don't need another timer thread. + """ + if not self.is_stale(): + return False + logger.info( + "Session idle for > %.0fs — auto-resetting", + self._idle_timeout_seconds, + ) + self.reset() + return True + + @staticmethod + def _validate_updates(updates: dict[str, Any]) -> None: + unknown = set(updates) - set(VisitorSession.__dataclass_fields__) + if unknown: + raise ValueError(f"Unknown session fields: {sorted(unknown)!r}") diff --git a/src/reachy_mini_receptionist/static/dashboard.html b/src/reachy_mini_receptionist/static/dashboard.html new file mode 100644 index 0000000000000000000000000000000000000000..40a00175b620a31ee6c9a2af0fe2c979165681b9 --- /dev/null +++ b/src/reachy_mini_receptionist/static/dashboard.html @@ -0,0 +1,2432 @@ + + + + + + MethdAI Receptionist + + + + + + + + +
+ + +

MethdAI Receptionist

+ + + + + +
+ + + — +
+ +
+
+ Scanning... + +
+
+ +
+ + + + +
Live
+ + +
+
+ Visits Today + — + Since midnight +
+
+ Emails Delivered + — + Hosts notified today +
+
+ Last Visit + — + — +
+
+ Current State + + idle + + No active visitor +
+
+ + +
+
+ 🎥Live Camera Feed + ● LIVE +
+
+
+ +
+ Camera feed +
Scanning...
+
+ + +
+
Best Face (5s)
+
+ 👤 +
+
—
+
No detection
+ +
+
🧠 Last Face Event Sent To Model
+
    +
  • +
    Status
    +
    No face event sent yet
    +
    • +
  • +
    Current
    +
    —
    +
    • +
  • +
    Previous
    +
    —
    +
    • +
+
+
+
+
+
+ + +
+
+ ⚡Active Session + +
+
+
+
+
State
+
+ idle +
+
+
+
Visitor
+
—
+
+
+
Host
+
—
+
+
+
Appointment
+
—
+
+ +
+ Journey + +
+ +
+
Latest cue sent to LLM
+
Waiting for a state change…
+
+
+
+
+ +
Today
+ + +
+
+ 📅Today's Schedule +
+
+
    +
  • Loading...
    • +
+
+
+ + +
+
+ 👥Known Guests + + + + +
+
+
    +
  • No guests registered yet
    • +
+
+
+ + +
+
+ 📧Mailbox Out + +
+
+
    +
  • No emails sent yet
    • +
+
+
+ + +
History
+ + +
+
+ 📋Visitor Log + + + ⬇ Export CSV + + +
+
+
+ + + + + + + + + + + + + + +
ArrivedVisitorHostAppointmentEmailOutcome
No visits logged yet
+
+
+
+ + +
+
+ 🧑‍💼Employees + + + + +
+
+
+ + + + + + + + + + + + + +
NameEmailTitleAliasesActions
Loading…
+
+
+
+ + +
+
+ ⚙️Settings + +
+
+
+
Loading…
+
+
+ + Secrets are masked. Edits write to .env. + RESTART = restart required. + + +
+ + +
+ Diagnostics (health checks · speaker test) +
+ + +
+
+
+
Loading…
+
+
+
+
+
+ +
+ + + + + +
+ + + + + + diff --git a/src/reachy_mini_receptionist/static/index.html b/src/reachy_mini_receptionist/static/index.html new file mode 100644 index 0000000000000000000000000000000000000000..d934aee60fd68c987d71c70acbcf560eefe4fb1f --- /dev/null +++ b/src/reachy_mini_receptionist/static/index.html @@ -0,0 +1,120 @@ + + + + + + Reachy Mini Conversation – Settings + + + +
+
+ +

Loading…

+
+
+ + +
+
Headless control
+

Reachy Mini Conversation

+

Configure your OpenAI key and tweak personalities without the full UI.

+
+ + + + + + +
+ + + + diff --git a/src/reachy_mini_receptionist/static/main.js b/src/reachy_mini_receptionist/static/main.js new file mode 100644 index 0000000000000000000000000000000000000000..4e4f5ecfdd917380148832798e13d951381e3686 --- /dev/null +++ b/src/reachy_mini_receptionist/static/main.js @@ -0,0 +1,496 @@ +async function fetchStatus() { + try { + const url = new URL("/status", window.location.origin); + url.searchParams.set("_", Date.now().toString()); + const resp = await fetchWithTimeout(url, {}, 2000); + if (!resp.ok) throw new Error("status error"); + return await resp.json(); + } catch (e) { + return { has_key: false, error: true }; + } +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function fetchWithTimeout(url, options = {}, timeoutMs = 2000) { + const controller = new AbortController(); + const id = setTimeout(() => controller.abort(), timeoutMs); + try { + return await fetch(url, { ...options, signal: controller.signal }); + } finally { + clearTimeout(id); + } +} + +async function waitForStatus(timeoutMs = 15000) { + const deadline = Date.now() + timeoutMs; + while (true) { + try { + const url = new URL("/status", window.location.origin); + url.searchParams.set("_", Date.now().toString()); + const resp = await fetchWithTimeout(url, {}, 2000); + if (resp.ok) return await resp.json(); + } catch (e) {} + if (Date.now() >= deadline) return null; + await sleep(500); + } +} + +async function waitForPersonalityData(timeoutMs = 15000) { + const loadingText = document.querySelector("#loading p"); + let attempts = 0; + const deadline = Date.now() + timeoutMs; + while (true) { + attempts += 1; + try { + const url = new URL("/personalities", window.location.origin); + url.searchParams.set("_", Date.now().toString()); + const resp = await fetchWithTimeout(url, {}, 2000); + if (resp.ok) return await resp.json(); + } catch (e) {} + + if (loadingText) { + loadingText.textContent = attempts > 8 ? "Starting backend…" : "Loading…"; + } + if (Date.now() >= deadline) return null; + await sleep(500); + } +} + +async function validateKey(key) { + const body = { openai_api_key: key }; + const resp = await fetch("/validate_api_key", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + const data = await resp.json().catch(() => ({})); + if (!resp.ok) { + throw new Error(data.error || "validation_failed"); + } + return data; +} + +async function saveKey(key) { + const body = { openai_api_key: key }; + const resp = await fetch("/openai_api_key", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!resp.ok) { + const data = await resp.json().catch(() => ({})); + throw new Error(data.error || "save_failed"); + } + return await resp.json(); +} + +// ---------- Personalities API ---------- +async function getPersonalities() { + const url = new URL("/personalities", window.location.origin); + url.searchParams.set("_", Date.now().toString()); + const resp = await fetchWithTimeout(url, {}, 2000); + if (!resp.ok) throw new Error("list_failed"); + return await resp.json(); +} + +async function loadPersonality(name) { + const url = new URL("/personalities/load", window.location.origin); + url.searchParams.set("name", name); + url.searchParams.set("_", Date.now().toString()); + const resp = await fetchWithTimeout(url, {}, 3000); + if (!resp.ok) throw new Error("load_failed"); + return await resp.json(); +} + +async function savePersonality(payload) { + // Try JSON POST first + const saveUrl = new URL("/personalities/save", window.location.origin); + saveUrl.searchParams.set("_", Date.now().toString()); + let resp = await fetchWithTimeout(saveUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }, 5000); + if (resp.ok) return await resp.json(); + + // Fallback to form-encoded POST + try { + const form = new URLSearchParams(); + form.set("name", payload.name || ""); + form.set("instructions", payload.instructions || ""); + form.set("tools_text", payload.tools_text || ""); + form.set("voice", payload.voice || "marin"); + const url = new URL("/personalities/save_raw", window.location.origin); + url.searchParams.set("_", Date.now().toString()); + resp = await fetchWithTimeout(url, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: form.toString(), + }, 5000); + if (resp.ok) return await resp.json(); + } catch {} + + // Fallback to GET (query params) + try { + const url = new URL("/personalities/save_raw", window.location.origin); + url.searchParams.set("name", payload.name || ""); + url.searchParams.set("instructions", payload.instructions || ""); + url.searchParams.set("tools_text", payload.tools_text || ""); + url.searchParams.set("voice", payload.voice || "marin"); + url.searchParams.set("_", Date.now().toString()); + resp = await fetchWithTimeout(url, { method: "GET" }, 5000); + if (resp.ok) return await resp.json(); + } catch {} + + const data = await resp.json().catch(() => ({})); + throw new Error(data.error || "save_failed"); +} + +async function applyPersonality(name, { persist = false } = {}) { + // Send as query param to avoid any body parsing issues on the server + const url = new URL("/personalities/apply", window.location.origin); + url.searchParams.set("name", name || ""); + if (persist) { + url.searchParams.set("persist", "1"); + } + url.searchParams.set("_", Date.now().toString()); + const resp = await fetchWithTimeout(url, { method: "POST" }, 5000); + if (!resp.ok) { + const data = await resp.json().catch(() => ({})); + throw new Error(data.error || "apply_failed"); + } + return await resp.json(); +} + +async function getVoices() { + try { + const url = new URL("/voices", window.location.origin); + url.searchParams.set("_", Date.now().toString()); + const resp = await fetchWithTimeout(url, {}, 3000); + if (!resp.ok) throw new Error("voices_failed"); + return await resp.json(); + } catch (e) { + return ["marin"]; + } +} + +function show(el, flag) { + el.classList.toggle("hidden", !flag); +} + +async function init() { + const loading = document.getElementById("loading"); + show(loading, true); + const statusEl = document.getElementById("status"); + const formPanel = document.getElementById("form-panel"); + const configuredPanel = document.getElementById("configured"); + const personalityPanel = document.getElementById("personality-panel"); + const saveBtn = document.getElementById("save-btn"); + const changeKeyBtn = document.getElementById("change-key-btn"); + const input = document.getElementById("api-key"); + + // Personality elements + const pSelect = document.getElementById("personality-select"); + const pApply = document.getElementById("apply-personality"); + const pPersist = document.getElementById("persist-personality"); + const pNew = document.getElementById("new-personality"); + const pSave = document.getElementById("save-personality"); + const pStartupLabel = document.getElementById("startup-label"); + const pName = document.getElementById("personality-name"); + const pInstr = document.getElementById("instructions-ta"); + const pTools = document.getElementById("tools-ta"); + const pStatus = document.getElementById("personality-status"); + const pVoice = document.getElementById("voice-select"); + const pAvail = document.getElementById("tools-available"); + + const AUTO_WITH = { + dance: ["stop_dance"], + play_emotion: ["stop_emotion"], + }; + + statusEl.textContent = "Checking configuration..."; + show(formPanel, false); + show(configuredPanel, false); + show(personalityPanel, false); + + const st = (await waitForStatus()) || { has_key: false }; + if (st.has_key) { + statusEl.textContent = ""; + show(configuredPanel, true); + } + + // Handler for "Change API key" button + changeKeyBtn.addEventListener("click", () => { + show(configuredPanel, false); + show(formPanel, true); + input.value = ""; + statusEl.textContent = ""; + statusEl.className = "status"; + }); + + // Remove error styling when user starts typing + input.addEventListener("input", () => { + input.classList.remove("error"); + }); + + saveBtn.addEventListener("click", async () => { + const key = input.value.trim(); + if (!key) { + statusEl.textContent = "Please enter a valid key."; + statusEl.className = "status warn"; + input.classList.add("error"); + return; + } + statusEl.textContent = "Validating API key..."; + statusEl.className = "status"; + input.classList.remove("error"); + try { + // First validate the key + const validation = await validateKey(key); + if (!validation.valid) { + statusEl.textContent = "Invalid API key. Please check your key and try again."; + statusEl.className = "status error"; + input.classList.add("error"); + return; + } + + // If valid, save it + statusEl.textContent = "Key valid! Saving..."; + statusEl.className = "status ok"; + await saveKey(key); + statusEl.textContent = "Saved. Reloading…"; + statusEl.className = "status ok"; + window.location.reload(); + } catch (e) { + input.classList.add("error"); + if (e.message === "invalid_api_key") { + statusEl.textContent = "Invalid API key. Please check your key and try again."; + } else { + statusEl.textContent = "Failed to validate/save key. Please try again."; + } + statusEl.className = "status error"; + } + }); + + if (!st.has_key) { + statusEl.textContent = ""; + show(formPanel, true); + show(loading, false); + return; + } + + // Wait until backend routes are ready before rendering personalities UI + const list = (await waitForPersonalityData()) || { choices: [] }; + statusEl.textContent = ""; + show(formPanel, false); + if (!list.choices.length) { + statusEl.textContent = "Personality endpoints not ready yet. Retry shortly."; + statusEl.className = "status warn"; + show(loading, false); + return; + } + + // Initialize personalities UI + try { + const choices = Array.isArray(list.choices) ? list.choices : []; + const DEFAULT_OPTION = choices[0] || "(built-in default)"; + const startupChoice = choices.includes(list.startup) ? list.startup : DEFAULT_OPTION; + const currentChoice = choices.includes(list.current) ? list.current : startupChoice; + + function setStartupLabel(name) { + const display = name && name !== DEFAULT_OPTION ? name : "Built-in default"; + pStartupLabel.textContent = `Launch on start: ${display}`; + } + + // Populate select + pSelect.innerHTML = ""; + for (const n of choices) { + const opt = document.createElement("option"); + opt.value = n; + opt.textContent = n; + pSelect.appendChild(opt); + } + if (choices.length) { + const preferred = choices.includes(startupChoice) ? startupChoice : currentChoice; + pSelect.value = preferred; + } + const voices = await getVoices(); + pVoice.innerHTML = ""; + for (const v of voices) { + const opt = document.createElement("option"); + opt.value = v; + opt.textContent = v; + pVoice.appendChild(opt); + } + setStartupLabel(startupChoice); + + function renderToolCheckboxes(available, enabled) { + pAvail.innerHTML = ""; + const enabledSet = new Set(enabled); + for (const t of available) { + const wrap = document.createElement("div"); + wrap.className = "chk"; + const id = `tool-${t}`; + const cb = document.createElement("input"); + cb.type = "checkbox"; + cb.id = id; + cb.value = t; + cb.checked = enabledSet.has(t); + const lab = document.createElement("label"); + lab.htmlFor = id; + lab.textContent = t; + wrap.appendChild(cb); + wrap.appendChild(lab); + pAvail.appendChild(wrap); + } + } + + function getSelectedTools() { + const selected = new Set(); + pAvail.querySelectorAll('input[type="checkbox"]').forEach((el) => { + if (el.checked) selected.add(el.value); + }); + // Auto-include dependencies + for (const [main, deps] of Object.entries(AUTO_WITH)) { + if (selected.has(main)) { + for (const d of deps) selected.add(d); + } + } + return Array.from(selected); + } + + function syncToolsTextarea() { + const selected = getSelectedTools(); + const comments = pTools.value + .split("\n") + .filter((ln) => ln.trim().startsWith("#")); + const body = selected.join("\n"); + pTools.value = (comments.join("\n") + (comments.length ? "\n" : "") + body).trim() + "\n"; + } + + function attachToolHandlers() { + pAvail.addEventListener("change", (ev) => { + const target = ev.target; + if (!(target instanceof HTMLInputElement) || target.type !== "checkbox") return; + const name = target.value; + // If a main tool toggled, propagate to deps + if (AUTO_WITH[name]) { + for (const dep of AUTO_WITH[name]) { + const depEl = pAvail.querySelector(`input[value="${dep}"]`); + if (depEl) depEl.checked = target.checked || depEl.checked; + } + } + syncToolsTextarea(); + }); + } + + async function loadSelected() { + const selected = pSelect.value; + const data = await loadPersonality(selected); + pInstr.value = data.instructions || ""; + pTools.value = data.tools_text || ""; + pVoice.value = data.voice || "marin"; + // Available tools as checkboxes + renderToolCheckboxes(data.available_tools, data.enabled_tools); + attachToolHandlers(); + // Default name field to last segment of selection + const idx = selected.lastIndexOf("/"); + pName.value = idx >= 0 ? selected.slice(idx + 1) : ""; + pStatus.textContent = `Loaded ${selected}`; + pStatus.className = "status"; + } + + pSelect.addEventListener("change", loadSelected); + await loadSelected(); + show(personalityPanel, true); + + // pAvail change handler registered in attachToolHandlers() + + pApply.addEventListener("click", async () => { + pStatus.textContent = "Applying..."; + pStatus.className = "status"; + try { + const res = await applyPersonality(pSelect.value); + if (res.startup) setStartupLabel(res.startup); + pStatus.textContent = res.status || "Applied."; + pStatus.className = "status ok"; + } catch (e) { + pStatus.textContent = `Failed to apply${e.message ? ": " + e.message : ""}`; + pStatus.className = "status error"; + } + }); + + pPersist.addEventListener("click", async () => { + pStatus.textContent = "Saving for startup..."; + pStatus.className = "status"; + try { + const res = await applyPersonality(pSelect.value, { persist: true }); + if (res.startup) setStartupLabel(res.startup); + pStatus.textContent = res.status || "Saved for startup."; + pStatus.className = "status ok"; + } catch (e) { + pStatus.textContent = `Failed to persist${e.message ? ": " + e.message : ""}`; + pStatus.className = "status error"; + } + }); + + pNew.addEventListener("click", () => { + pName.value = ""; + pInstr.value = "# Write your instructions here\n# e.g., Keep responses concise and friendly."; + pTools.value = "# tools enabled for this profile\n"; + // Keep available tools list, clear selection + pAvail.querySelectorAll('input[type="checkbox"]').forEach((el) => { + el.checked = false; + }); + pVoice.value = "marin"; + pStatus.textContent = "Fill fields and click Save."; + pStatus.className = "status"; + }); + + pSave.addEventListener("click", async () => { + const name = (pName.value || "").trim(); + if (!name) { + pStatus.textContent = "Enter a valid name."; + pStatus.className = "status warn"; + return; + } + pStatus.textContent = "Saving..."; + pStatus.className = "status"; + try { + // Ensure tools.txt reflects checkbox selection and auto-includes + syncToolsTextarea(); + const res = await savePersonality({ + name, + instructions: pInstr.value || "", + tools_text: pTools.value || "", + voice: pVoice.value || "marin", + }); + // Refresh select choices + pSelect.innerHTML = ""; + for (const n of res.choices) { + const opt = document.createElement("option"); + opt.value = n; + opt.textContent = n; + if (n === res.value) opt.selected = true; + pSelect.appendChild(opt); + } + pStatus.textContent = "Saved."; + pStatus.className = "status ok"; + // Auto-apply + try { await applyPersonality(pSelect.value); } catch {} + } catch (e) { + pStatus.textContent = "Failed to save."; + pStatus.className = "status error"; + } + }); + } catch (e) { + statusEl.textContent = "UI failed to load. Please refresh."; + statusEl.className = "status warn"; + } finally { + // Hide loading when initial setup is done (regardless of key presence) + show(loading, false); + } +} + +window.addEventListener("DOMContentLoaded", init); diff --git a/src/reachy_mini_receptionist/static/style.css b/src/reachy_mini_receptionist/static/style.css new file mode 100644 index 0000000000000000000000000000000000000000..53916a0ff8c9997e38957a4fb590bcd12f6a6f14 --- /dev/null +++ b/src/reachy_mini_receptionist/static/style.css @@ -0,0 +1,340 @@ +:root { + --bg: #060b1a; + --bg-2: #071023; + --panel: rgba(11, 18, 36, 0.8); + --border: rgba(255, 255, 255, 0.08); + --text: #eaf2ff; + --muted: #9fb6d7; + --ok: #4ce0b3; + --warn: #ffb547; + --error: #ff5c70; + --accent: #45c4ff; + --accent-2: #5ef0c1; + --shadow: 0 20px 70px rgba(0, 0, 0, 0.45); +} + +* { box-sizing: border-box; } +body { + margin: 0; + min-height: 100vh; + font-family: "Space Grotesk", "Inter", "Segoe UI", sans-serif; + background: radial-gradient(circle at 20% 20%, rgba(69, 196, 255, 0.16), transparent 35%), + radial-gradient(circle at 80% 0%, rgba(94, 240, 193, 0.16), transparent 32%), + linear-gradient(135deg, var(--bg), var(--bg-2)); + color: var(--text); +} + +.ambient { + position: fixed; + inset: 0; + background: radial-gradient(circle at 30% 60%, rgba(255, 255, 255, 0.05), transparent 35%), + radial-gradient(circle at 75% 30%, rgba(69, 196, 255, 0.08), transparent 32%); + filter: blur(60px); + z-index: 0; + pointer-events: none; +} + +/* Loading overlay */ +.loading { + position: fixed; + inset: 0; + background: rgba(5, 10, 24, 0.92); + backdrop-filter: blur(4px); + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + z-index: 9999; +} +.loading .spinner { + width: 46px; + height: 46px; + border: 4px solid rgba(255,255,255,0.15); + border-top-color: var(--accent); + border-radius: 50%; + animation: spin 1s linear infinite; + margin-bottom: 12px; +} +.loading p { color: var(--muted); margin: 0; letter-spacing: 0.4px; } +@keyframes spin { to { transform: rotate(360deg); } } + +.container { + position: relative; + max-width: 960px; + margin: 7vh auto; + padding: 0 24px 40px; + z-index: 1; +} + +.dashboard-banner { + margin-bottom: 14px; +} + +.dashboard-banner a { + display: block; + width: 100%; + text-align: center; + font-size: 1.15rem; + font-weight: 700; + letter-spacing: 0.2px; + padding: 14px 16px; + border-radius: 12px; + border: 1px solid rgba(94, 240, 193, 0.45); + color: #041127; + background: linear-gradient(120deg, var(--accent), var(--accent-2)); + box-shadow: 0 14px 35px rgba(69, 196, 255, 0.25); +} + +.dashboard-banner a:hover { + filter: brightness(1.05); +} + +.hero { + margin-bottom: 24px; +} +.hero h1 { + margin: 6px 0 6px; + font-size: 32px; + letter-spacing: -0.4px; +} +.subtitle { + margin: 0; + color: var(--muted); + line-height: 1.5; +} +.pill { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 6px 12px; + border-radius: 999px; + background: rgba(94, 240, 193, 0.1); + color: var(--accent-2); + font-size: 12px; + letter-spacing: 0.3px; + border: 1px solid rgba(94, 240, 193, 0.25); +} + +.panel { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 14px; + padding: 18px 18px 16px; + box-shadow: var(--shadow); + backdrop-filter: blur(10px); + margin-top: 16px; +} +.panel-heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + margin-bottom: 8px; +} +.panel-heading h2 { + margin: 2px 0; + font-size: 22px; +} +.eyebrow { + margin: 0; + text-transform: uppercase; + font-size: 11px; + letter-spacing: 0.5px; + color: var(--muted); +} +.muted { color: var(--muted); } +.chip { + display: inline-flex; + align-items: center; + padding: 6px 10px; + border-radius: 999px; + font-size: 12px; + color: var(--text); + background: rgba(255, 255, 255, 0.08); + border: 1px solid var(--border); +} +.chip-ok { + background: rgba(76, 224, 179, 0.15); + color: var(--ok); + border-color: rgba(76, 224, 179, 0.4); +} + +.hidden { display: none; } +label { + display: block; + margin: 8px 0 6px; + font-size: 13px; + color: var(--muted); + letter-spacing: 0.2px; +} +input[type="password"], +input[type="text"], +select, +textarea { + width: 100%; + padding: 12px 14px; + border: 1px solid var(--border); + border-radius: 10px; + background: rgba(255, 255, 255, 0.04); + color: var(--text); + transition: border 0.15s ease, box-shadow 0.15s ease; +} +input:focus, +select:focus, +textarea:focus { + border-color: rgba(94, 240, 193, 0.7); + outline: none; + box-shadow: 0 0 0 3px rgba(94, 240, 193, 0.15); +} +input.error { + border-color: var(--error); + box-shadow: 0 0 0 3px rgba(255, 92, 112, 0.15); +} +select option { + background: #0b152a; + color: var(--text); +} +textarea { resize: vertical; } + +button { + display: inline-flex; + align-items: center; + justify-content: center; + margin-top: 12px; + padding: 11px 16px; + border: none; + border-radius: 10px; + background: linear-gradient(120deg, var(--accent), var(--accent-2)); + color: #031022; + cursor: pointer; + font-weight: 600; + letter-spacing: 0.2px; + box-shadow: 0 14px 40px rgba(69, 196, 255, 0.25); + transition: transform 0.12s ease, filter 0.12s ease, box-shadow 0.12s ease; +} +button:hover { filter: brightness(1.06); transform: translateY(-1px); } +button:active { transform: translateY(0); } +button.ghost { + background: rgba(255, 255, 255, 0.05); + color: var(--text); + box-shadow: none; + border: 1px solid var(--border); +} +button.ghost:hover { border-color: rgba(94, 240, 193, 0.4); } +.actions { + display: flex; + align-items: center; + gap: 12px; + flex-wrap: wrap; +} +.status { + margin: 0; + color: var(--muted); + font-size: 13px; +} +.status.ok { color: var(--ok); } +.status.warn { color: var(--warn); } +.status.error { color: var(--error); } + +/* Personality layout */ +.row { + display: grid; + grid-template-columns: 160px 1fr; + gap: 12px 18px; + align-items: center; + margin-top: 12px; +} +.row > label { margin: 0; } +.row > button { margin: 0; } + +/* First row: controls inline */ +#personality-panel .row-top { + grid-template-columns: 160px 1fr auto auto auto; +} + +#tools-available { + max-height: 240px; + overflow: auto; + padding: 10px; + border: 1px solid var(--border); + border-radius: 10px; + background: rgba(255, 255, 255, 0.03); +} + +/* Checkbox grid for tools */ +.checkbox-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(170px, 1fr)); + gap: 10px 14px; +} +.startup-row { + display: flex; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} +.row-save .actions { + justify-content: flex-start; +} +.input-field { + width: 100%; + padding: 12px 14px; + border: 1px solid var(--border); + border-radius: 10px; + background: rgba(255, 255, 255, 0.05); + color: var(--text); + transition: border 0.15s ease, box-shadow 0.15s ease; +} +.input-field:focus { + border-color: rgba(94, 240, 193, 0.7); + outline: none; + box-shadow: 0 0 0 3px rgba(94, 240, 193, 0.15); +} +.section { + border: 1px solid var(--border); + border-radius: 12px; + padding: 12px 14px; + margin-top: 14px; + background: rgba(255, 255, 255, 0.02); +} +.section-heading { + display: flex; + align-items: baseline; + gap: 10px; + justify-content: space-between; +} +.section-heading h3 { + margin: 6px 0; + font-size: 16px; + letter-spacing: -0.1px; +} +.section-heading .small { + margin: 0; + font-size: 12px; +} +.checkbox-grid .chk { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 10px; + border-radius: 10px; + background: rgba(255, 255, 255, 0.02); + border: 1px solid transparent; + transition: border 0.12s ease, background 0.12s ease; +} +.checkbox-grid .chk:hover { border-color: rgba(94, 240, 193, 0.3); background: rgba(255, 255, 255, 0.04); } +.checkbox-grid input[type="checkbox"] { + width: 16px; height: 16px; + accent-color: var(--accent); +} +.checkbox-grid label { + margin: 0; font-size: 13px; color: var(--text); +} + +@media (max-width: 760px) { + .hero h1 { font-size: 26px; } + .row { grid-template-columns: 1fr; } + #personality-panel .row:first-of-type { grid-template-columns: 1fr; } + button { width: 100%; justify-content: center; } + .actions { flex-direction: column; align-items: flex-start; } +} diff --git a/src/reachy_mini_receptionist/tools/__init__.py b/src/reachy_mini_receptionist/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e8e6e4b40030c0eb2c5a815b3208019317d935fc --- /dev/null +++ b/src/reachy_mini_receptionist/tools/__init__.py @@ -0,0 +1,4 @@ +"""Tools library for Reachy Mini receptionist app. + +Tools are now loaded dynamically based on the profile's tools.txt file. +""" diff --git a/src/reachy_mini_receptionist/tools/background_tool_manager.py b/src/reachy_mini_receptionist/tools/background_tool_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..93b54c63da129aebc53c0cd0ba373259f68ff401 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/background_tool_manager.py @@ -0,0 +1,418 @@ +"""Background tool orchestrator for non-blocking tool execution. + +Allows tools to run long operations asynchronously while the robot +continues conversing. Tools can be tracked, cancelled, and their +completion is announced vocally via a silent notification queue. +""" + +# NOTE: ``from __future__ import annotations`` was removed here because +# Python 3.12 on the robot intermittently raised a "from __future__ +# imports must occur at the beginning of the file" SyntaxError against +# this file even after CRLF was cleaned. Python 3.12 already evaluates +# annotations lazily where needed; removing the import has no runtime +# effect on this module. + +import time +import asyncio +import logging +from typing import Any, Dict, Callable, Optional, Coroutine + +from pydantic import Field, BaseModel, PrivateAttr + +from reachy_mini_receptionist.tools.core_tools import ( + ToolDependencies, + dispatch_tool_call, + dispatch_tool_call_with_manager, +) +from reachy_mini_receptionist.tools.tool_constants import ToolState, SystemTool + + +logger = logging.getLogger(__name__) + +_SYSTEM_TOOL_NAMES: set[str] = {t.value for t in SystemTool} + +class ToolProgress(BaseModel): + """Progress of a background tool.""" + + """the progress of the tool""" + progress: float = Field(..., ge=0.0, le=1.0) + + """the message of the tool""" + message: Optional[str] = None + + +class ToolCallRoutine(BaseModel): + """Encapsulates an async callable with its arguments for deferred execution.""" + + model_config = {"arbitrary_types_allowed": True} + + """the name of the tool""" + tool_name: str + + """the JSON arguments for the tool call""" + args_json_str: str + + """the dependencies for the tool call""" + deps: "ToolDependencies" + + async def __call__(self, tool_manager: "BackgroundToolManager") -> Any: + """Execute the stored callable with its arguments.""" + if self.tool_name in _SYSTEM_TOOL_NAMES: + # For safety purposes, we only allow system tools to be called with the tool manager + return await dispatch_tool_call_with_manager(tool_name=self.tool_name, args_json=self.args_json_str, deps=self.deps, tool_manager=tool_manager) + return await dispatch_tool_call(tool_name=self.tool_name, args_json=self.args_json_str, deps=self.deps) + + +class ToolNotification(BaseModel): + """Notification payload for completed tools.""" + + """the ID of the tool""" + id: str + + """the name of the tool""" + tool_name: str + + """whether the tool call was triggered by an idle signal""" + is_idle_tool_call: bool + + """the status of the tool""" + status: ToolState + + """the result of the tool""" + result: Optional[Dict[str, Any]] = None + + """the error of the tool""" + error: Optional[str] = None + + +class BackgroundTool(ToolNotification): + """Represents a background tool.""" + + """the progress of the tool""" + progress: Optional[ToolProgress] = None + + """the start time of the tool""" + started_at: float = Field(default_factory=time.monotonic) + + """the completion time of the tool""" + completed_at: Optional[float] = None + + """the async tool execution task""" + _task: Optional[asyncio.Task[None]] = PrivateAttr(default=None) + + @property + def tool_id(self) -> str: + """Get the name of the tool.""" + return f"{self.tool_name}-{self.id}-{self.started_at}" + + def get_notification(self) -> ToolNotification: + """Get the notification for the tool.""" + return ToolNotification( + id=self.id, + tool_name=self.tool_name, + is_idle_tool_call=self.is_idle_tool_call, + status=self.status, + result=self.result, + error=self.error, + ) + + +class BackgroundToolManager(BaseModel): + """Manages background tools for non-blocking tool execution. + + Features: + - Start async tools without blocking the conversation + - Track tool status and progress + - Cancel running tools + + """ + + """the dictionary of tools""" + _tools: Dict[str, BackgroundTool] = PrivateAttr(default_factory=dict) + + """the async queue for notifications""" + _notification_queue: asyncio.Queue[ToolNotification] = PrivateAttr(default_factory=asyncio.Queue) + + """the event loop""" + _loop: Optional[asyncio.AbstractEventLoop] = PrivateAttr(default=None) + + """internal lifecycle tasks (notification listener, periodic cleanup)""" + _lifecycle_tasks: list[asyncio.Task[None]] = PrivateAttr(default_factory=list) + + """the maximum duration of a tool execution in seconds (default: 1 day)""" + _max_tool_duration_seconds: float = PrivateAttr(default=86400) + + """the maximum time to keep a completed/failed/cancelled tool in memory (default: 1 hour)""" + _max_tool_memory_seconds: float = PrivateAttr(default=3600) + + def set_loop( + self, + loop: Optional[asyncio.AbstractEventLoop] = None, + ) -> None: + """Set the event loop. + + Args: + loop: The event loop (defaults to current running loop) + + """ + if loop is not None: + self._loop = loop + else: + try: + self._loop = asyncio.get_running_loop() + except RuntimeError: + self._loop = asyncio.new_event_loop() + logger.debug("BackgroundToolManager: event loop set") + + + async def start_tool( + self, + call_id: str, + tool_call_routine: ToolCallRoutine, + is_idle_tool_call: bool, + with_progress: bool = False, + ) -> BackgroundTool: + """Start a new background tool. + + Args: + call_id: The ID of the tool + tool_call_routine: The ToolCallRoutine containing the callable and its arguments + with_progress: Whether to track progress (0.0-1.0) + is_idle_tool_call: Whether the tool call was triggered by an idle signal + + Returns: + BackgroundTool object with tool ID + + """ + tool_name = tool_call_routine.tool_name + id = call_id + bg_tool = BackgroundTool( + id=id, + tool_name=tool_name, + is_idle_tool_call=is_idle_tool_call, + progress=ToolProgress(progress=0.0) if with_progress else None, + status=ToolState.RUNNING, + ) + self._tools[bg_tool.tool_id] = bg_tool + + async_task = asyncio.create_task( + self._run_tool(bg_tool, tool_call_routine), + name=f"bg-{tool_name}-{id}", + ) + bg_tool._task = async_task + + logger.info(f"Started background tool: {bg_tool.tool_name} (id={id})") + + return bg_tool + + async def _run_tool( + self, + bg_tool: BackgroundTool, + tool_call_routine: ToolCallRoutine, + ) -> None: + """Execute the tool and handle completion.""" + result: dict[str, Any] = await tool_call_routine(self) + bg_tool.completed_at = time.monotonic() + error = result.get("error") + + if error is not None: + if error == "Tool cancelled": + bg_tool.status = ToolState.CANCELLED + logger.debug(f"Background tool cancelled: {bg_tool.tool_name} (id={bg_tool.id})") + else: + bg_tool.status = ToolState.FAILED + logger.debug(f"Background tool failed: {bg_tool.tool_name} (id={bg_tool.id}): {bg_tool.error}") + bg_tool.error = result["error"] + + else: + bg_tool.result = result + bg_tool.status = ToolState.COMPLETED + logger.debug(f"Background tool completed: {bg_tool.tool_name} (id={bg_tool.id})") + + await self._notification_queue.put(bg_tool.get_notification()) + logger.debug(f"Queued notification for tool: {bg_tool.tool_name} (id={bg_tool.id})") + + async def update_progress( + self, + tool_id: str, + progress: float, + message: Optional[str] = None, + ) -> bool: + """Update progress for a tool (for tools with with_progress=True). + + Args: + tool_id: The tool ID + progress: Progress value between 0.0 and 1.0 + message: Optional progress message (e.g., "50% downloaded") + + Returns: + True if updated successfully, False if tool not found or not tracking progress + + """ + tool = self._tools.get(tool_id) + if tool is None: + return False + + if tool.progress is None: + # Tool not tracking progress + return False + + tool.progress = ToolProgress(progress=max(0.0, min(1.0, progress)), message=message) + logger.debug(f"Tool {tool_id} progress: {progress:.1%} - {message or ''}") + return True + + async def cancel_tool(self, tool_id: str, log: bool = True) -> bool: + """Cancel a running tool by ID. + + Args: + tool_id: The tool ID to cancel + log: Whether to log the cancellation + + Returns: + True if cancelled, False if tool not found or not running + + """ + tool = self._tools.get(tool_id) + if tool is None: + if log: + logger.warning(f"Cannot cancel tool {tool_id}: not found") + return False + + if tool.status != ToolState.RUNNING: + if log: + logger.warning(f"Cannot cancel tool {tool_id}: status is {tool.status.value}") + return True + + if tool._task: + tool._task.cancel() + if log: + logger.info(f"Cancelled tool: {tool.tool_name} (id={tool_id})") + return True + + return False + + def start_up(self, tool_callbacks: list[Callable[[ToolNotification], Coroutine[Any, Any, None]]]) -> None: + """Start the background tool manager. + + This method starts two concurrent tasks: + - _listener: Listens for completed BackgroundTool notifications and calls the callbacks. + - _cleanup: Cleans up completed/failed/cancelled tools that have been in memory for too long and times out tools that have been running too long. + + Args: + tool_callbacks: A list of async or sync callables that receive the completed BackgroundTool notifications. + + """ + self.set_loop() + + async def _listener() -> None: + while True: + bg_tool = await self._notification_queue.get() + for callback in tool_callbacks: + await callback(bg_tool) + + async def _cleanup(interval_seconds: float = 5 * 60) -> None: + while True: + await asyncio.sleep(interval_seconds) + await self.cleanup_tools() + await self.timeout_tools() + + self._lifecycle_tasks = [ + asyncio.create_task(_cleanup(), name="bg-tool-cleanup"), + asyncio.create_task(_listener(), name="bg-tool-listener-callback"), + ] + + logger.info( + "BackgroundToolManager started. " + "Max tool execution duration: %s seconds (tools running longer will be auto-cancelled). " + "Max tool memory retention: %s seconds (completed/failed/cancelled tools older than this are purged).", + self._max_tool_duration_seconds, self._max_tool_memory_seconds, + ) + + async def shutdown(self) -> None: + """Cancel all background tasks (listener, cleanup) and running tools.""" + for task in self._lifecycle_tasks: + task.cancel() + for task in self._lifecycle_tasks: + try: + await task + except asyncio.CancelledError: + pass + self._lifecycle_tasks.clear() + + for tool_id in list(self._tools): + await self.cancel_tool(tool_id, log=False) + + logger.info("BackgroundToolManager shut down") + + async def timeout_tools(self) -> int: + """Cancel tools that have been running too long. + + Returns: + Number of tools cancelled + + """ + now = time.monotonic() + to_cancel = [] + + for tool_id, tool in self._tools.items(): + if tool.status == ToolState.RUNNING: + if tool.started_at and (now - tool.started_at) > self._max_tool_duration_seconds: + to_cancel.append(tool_id) + + for tool_id in to_cancel: + await self.cancel_tool(tool_id) + + if to_cancel: + logger.debug(f"Timed out {len(to_cancel)} tools") + + return len(to_cancel) + + async def cleanup_tools(self) -> int: + """Remove completed/failed/cancelled tools that have been in memory for too long. + + Returns: + Number of tools removed + + """ + now = time.monotonic() + to_remove = [] + + for tool_id, tool in self._tools.items(): + if tool.status in (ToolState.COMPLETED, ToolState.FAILED, ToolState.CANCELLED): + if tool.completed_at and (now - tool.completed_at) > self._max_tool_memory_seconds: + to_remove.append(tool_id) + + for tool_id in to_remove: + del self._tools[tool_id] + + if to_remove: + logger.debug(f"Cleaned up {len(to_remove)} old tools") + + return len(to_remove) + + def get_tool(self, tool_id: str) -> Optional[BackgroundTool]: + """Get a tool by ID.""" + return self._tools.get(tool_id) + + def get_running_tools(self) -> list[BackgroundTool]: + """Get all currently running tools.""" + return [t for t in self._tools.values() if t.status == ToolState.RUNNING] + + def get_all_tools(self, limit: Optional[int] = None) -> list[BackgroundTool]: + """Get recent tools (most recent first). + + Args: + limit: Maximum number of tools to return (None means all) + + Returns: + List of tools sorted by start time (most recent first) + + """ + sorted_tools = sorted( + self._tools.values(), + key=lambda t: t.started_at, + reverse=True, + ) + if limit is not None: + return sorted_tools[:limit] + return sorted_tools diff --git a/src/reachy_mini_receptionist/tools/camera.py b/src/reachy_mini_receptionist/tools/camera.py new file mode 100644 index 0000000000000000000000000000000000000000..29da9dff13e887b9d2bed28f9821c5aa8fe21c5f --- /dev/null +++ b/src/reachy_mini_receptionist/tools/camera.py @@ -0,0 +1,68 @@ +import base64 +import asyncio +import logging +from typing import Any, Dict + +import cv2 + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + + +class Camera(Tool): + """Take a picture with the camera and ask a question about it.""" + + name = "camera" + description = "Take a picture with the camera and ask a question about it." + parameters_schema = { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": "The question to ask about the picture", + }, + }, + "required": ["question"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Take a picture with the camera and ask a question about it.""" + image_query = (kwargs.get("question") or "").strip() + if not image_query: + logger.warning("camera: empty question") + return {"error": "question must be a non-empty string"} + + logger.info("Tool call: camera question=%s", image_query[:120]) + + # Get frame from camera worker buffer (like main_works.py) + if deps.camera_worker is not None: + frame = deps.camera_worker.get_latest_frame() + if frame is None: + logger.error("No frame available from camera worker") + return {"error": "No frame available"} + else: + logger.error("Camera worker not available") + return {"error": "Camera worker not available"} + + # Use vision manager for processing if available + if deps.vision_manager is not None: + vision_result = await asyncio.to_thread( + deps.vision_manager.processor.process_image, frame, image_query, + ) + if isinstance(vision_result, dict) and "error" in vision_result: + return vision_result + return ( + {"image_description": vision_result} + if isinstance(vision_result, str) + else {"error": "vision returned non-string"} + ) + + # Encode image directly to JPEG bytes without writing to file + success, buffer = cv2.imencode('.jpg', frame) + if not success: + raise RuntimeError("Failed to encode frame as JPEG") + + b64_encoded = base64.b64encode(buffer.tobytes()).decode("utf-8") + return {"b64_im": b64_encoded} diff --git a/src/reachy_mini_receptionist/tools/check_current_face.py b/src/reachy_mini_receptionist/tools/check_current_face.py new file mode 100644 index 0000000000000000000000000000000000000000..0bb8904583e92d18345c5016dfdfdc5f20ba8fd2 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/check_current_face.py @@ -0,0 +1,49 @@ +"""Tool: check_current_face + +Returns the name of the person currently detected in front of the camera, +or "Unknown" if no match is found in the database. + +Instead of reading the instantaneous snapshot, this tool uses +`best_recent_face()` — which scans the last 5 seconds of detections and +picks the crop with the highest confidence before running LBPH +recognition. This avoids false "Unknown" results caused by the face briefly +disappearing from the frame at the exact moment the LLM calls this tool. +""" +from __future__ import annotations + +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +class CheckCurrentFace(Tool): + """Check who is currently in front of the camera.""" + + name = "check_current_face" + description = ( + "Returns the name of the person currently detected by the camera, " + "or 'Unknown' if the person hasn't been here before. " + ) + parameters_schema: Dict[str, Any] = { + "type": "object", + "properties": {}, + "required": [], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + # The face worker is stored on vision_processor slot of ToolDependencies + worker = getattr(deps, "face_worker", None) or getattr(deps, "vision_processor", None) + + if worker is None: + return {"name": "Unknown", "confidence": 0.0, "note": "Face recognition not available"} + + # Use the 5-second rolling window: picks the most confidently detected + # face seen recently, rather than a momentary snapshot. + if hasattr(worker, "best_recent_face"): + name, confidence, _crop = worker.best_recent_face(window_seconds=5.0) + + return { + "name": name, + "confidence": round(confidence, 2), + "is_known": name != "Unknown" and name != "No face", + } diff --git a/src/reachy_mini_receptionist/tools/core_tools.py b/src/reachy_mini_receptionist/tools/core_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..223be253f534b4049b6e5ea905a8f2f820e784dd --- /dev/null +++ b/src/reachy_mini_receptionist/tools/core_tools.py @@ -0,0 +1,335 @@ +from __future__ import annotations +import re +import abc +import sys +import json +import asyncio +import inspect +import logging +import importlib +import importlib.util +from typing import TYPE_CHECKING, Any, Dict, List +from pathlib import Path +from dataclasses import dataclass + +from reachy_mini import ReachyMini +from reachy_mini_receptionist.config import DEFAULT_PROFILES_DIRECTORY as DEFAULT_PROFILES_PATH # noqa: F401 + +# Import config to ensure .env is loaded before reading REACHY_MINI_CUSTOM_PROFILE +from reachy_mini_receptionist.config import config # noqa: F401 +from reachy_mini_receptionist.tools.tool_constants import SystemTool + + +if TYPE_CHECKING: + from reachy_mini_receptionist.tools.background_tool_manager import BackgroundToolManager + + +logger = logging.getLogger(__name__) + + +DEFAULT_PROFILES_MODULE = "reachy_mini_receptionist.profiles" + + +if not logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s:%(lineno)d | %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + + +ALL_TOOLS: Dict[str, "Tool"] = {} +ALL_TOOL_SPECS: List[Dict[str, Any]] = [] +_TOOLS_INITIALIZED = False + + + +def get_concrete_subclasses(base: type[Tool]) -> List[type[Tool]]: + """Recursively find all concrete (non-abstract) subclasses of a base class.""" + result: List[type[Tool]] = [] + for cls in base.__subclasses__(): + if not inspect.isabstract(cls): + result.append(cls) + # recurse into subclasses + result.extend(get_concrete_subclasses(cls)) + return result + + +@dataclass +class ToolDependencies: + """External dependencies injected into tools.""" + + reachy_mini: ReachyMini + movement_manager: Any # MovementManager from moves.py + # Optional deps + camera_worker: Any | None = None # CameraWorker for frame buffering + vision_manager: Any | None = None + head_wobbler: Any | None = None # HeadWobbler for audio-reactive motion + motion_duration_s: float = 1.0 + # Receptionist-specific deps + face_worker: Any | None = None # FaceRecognitionWorker + face_db: Any | None = None # FaceDatabase + session_manager: Any | None = None # SessionManager + conversation_controller: Any | None = None # ConversationController + + +# Tool base class +class Tool(abc.ABC): + """Base abstraction for tools used in function-calling. + + Each tool must define: + - name: str + - description: str + - parameters_schema: Dict[str, Any] # JSON Schema + """ + + name: str + description: str + parameters_schema: Dict[str, Any] + + def spec(self) -> Dict[str, Any]: + """Return the function spec for LLM consumption.""" + return { + "type": "function", + "name": self.name, + "description": self.description, + "parameters": self.parameters_schema, + } + + @abc.abstractmethod + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Async tool execution entrypoint.""" + raise NotImplementedError + + +def _load_module_from_file(module_name: str, file_path: Path) -> None: + """Load a Python module from a file path.""" + spec = importlib.util.spec_from_file_location(module_name, file_path) + if not (spec and spec.loader): + raise ModuleNotFoundError(f"Cannot create spec for {file_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + + +def _try_load_tool( + tool_name: str, + module_path: str, + fallback_directory: Path | None, + file_subpath: str, +) -> str: + """Try to load a tool: first via importlib, then from file if fallback is configured.""" + try: + importlib.import_module(module_path) + return "module" + except ModuleNotFoundError: + if fallback_directory is None: + raise + tool_file = fallback_directory / file_subpath + if not tool_file.exists(): + raise FileNotFoundError(f"tool file not found at {tool_file}") + _load_module_from_file(tool_name, tool_file) + return "file" + + +def _format_error(error: Exception) -> str: + """Format an exception for logging.""" + if isinstance(error, FileNotFoundError): + return f"Tool file not found: {error}" + if isinstance(error, ModuleNotFoundError): + return f"Missing dependency: {error}" + if isinstance(error, ImportError): + return f"Import error: {error}" + return f"{type(error).__name__}: {error}" + + +# Registry & specs (dynamic) +def _load_profile_tools() -> None: + """Load tools based on profile's tools.txt file.""" + # Determine which profile to use + profile = config.REACHY_MINI_CUSTOM_PROFILE or "default" + logger.info(f"Loading tools for profile: {profile}") + + # Build path to tools.txt + # Get the profile directory path + profile_module_path = config.PROFILES_DIRECTORY / profile + tools_txt_path = profile_module_path / "tools.txt" + default_tools_txt_path = Path(__file__).parent.parent / "profiles" / "default" / "tools.txt" + + if config.PROFILES_DIRECTORY != DEFAULT_PROFILES_PATH: + logger.info( + "Loading external profile '%s' from %s", + profile, + profile_module_path, + ) + + if not tools_txt_path.exists(): + if profile != "default" and default_tools_txt_path.exists(): + logger.warning( + "tools.txt not found for profile '%s' at %s. Falling back to default profile tools at %s", + profile, + tools_txt_path, + default_tools_txt_path, + ) + tools_txt_path = default_tools_txt_path + else: + logger.error(f"✗ tools.txt not found at {tools_txt_path}") + sys.exit(1) + + # Read and parse tools.txt + try: + with open(tools_txt_path, "r") as f: + lines = f.readlines() + except Exception as e: + logger.error(f"✗ Failed to read tools.txt: {e}") + sys.exit(1) + + # Parse tool names (skip comments and blank lines) + tool_names = [] + for line in lines: + line = line.strip() + # Skip blank lines and comments + if not line or line.startswith("#"): + continue + tool_names.append(line) + + # Add system tools + tool_names.extend({tool.value for tool in SystemTool}) + + logger.info(f"Found {len(tool_names)} tools to load: {tool_names}") + + if config.AUTOLOAD_EXTERNAL_TOOLS and config.TOOLS_DIRECTORY and config.TOOLS_DIRECTORY.is_dir(): + discovered_external_tools: List[str] = [] + for tool_file in sorted(config.TOOLS_DIRECTORY.glob("*.py")): + if tool_file.name.startswith("_"): + continue + candidate_name = tool_file.stem + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", candidate_name): + logger.warning("Skipping external tool with invalid name: %s", tool_file.name) + continue + discovered_external_tools.append(candidate_name) + + extra_tools = [name for name in discovered_external_tools if name not in tool_names] + if extra_tools: + tool_names.extend(extra_tools) + logger.info( + "AUTOLOAD_EXTERNAL_TOOLS enabled: added %d external tool(s): %s", + len(extra_tools), + extra_tools, + ) + + for tool_name in tool_names: + loaded = False + profile_error = None + profile_import_path = f"{DEFAULT_PROFILES_MODULE}.{profile}.{tool_name}" + + # Try profile tool first + try: + source = _try_load_tool( + tool_name, + module_path=profile_import_path, + fallback_directory=config.PROFILES_DIRECTORY, + file_subpath=f"{profile}/{tool_name}.py", + ) + if source == "file": + logger.info("✓ Loaded external profile tool: %s", tool_name) + else: + logger.info("✓ Loaded core profile tool: %s", tool_name) + loaded = True + except (ModuleNotFoundError, FileNotFoundError) as e: + if tool_name not in str(e): + profile_error = _format_error(e) + logger.error(f"❌ Failed to load profile tool '{tool_name}': {profile_error}") + logger.error(f" Module path: {profile_import_path}") + except Exception as e: + profile_error = _format_error(e) + logger.error(f"❌ Failed to load profile tool '{tool_name}': {profile_error}") + logger.error(f" Module path: {profile_import_path}") + + # Try tools directory if not found in profile + if not loaded: + shared_module_path = f"reachy_mini_receptionist.tools.{tool_name}" + try: + source = _try_load_tool( + tool_name, + module_path=shared_module_path, + fallback_directory=config.TOOLS_DIRECTORY, + file_subpath=f"{tool_name}.py", + ) + if source == "file": + logger.info("✓ Loaded external tool: %s", tool_name) + else: + logger.info("✓ Loaded core tool: %s", tool_name) + except (ModuleNotFoundError, FileNotFoundError): + if profile_error: + logger.error(f"❌ Tool '{tool_name}' also not found in shared tools") + else: + logger.warning(f"⚠️ Tool '{tool_name}' not found in profile or shared tools") + except Exception as e: + logger.error(f"❌ Failed to load shared tool '{tool_name}': {_format_error(e)}") + logger.error(f" Module path: {shared_module_path}") + + + +def _initialize_tools() -> None: + """Populate registry once, even if module is imported repeatedly.""" + global ALL_TOOLS, ALL_TOOL_SPECS, _TOOLS_INITIALIZED + + if _TOOLS_INITIALIZED: + logger.debug("Tools already initialized; skipping reinitialization.") + return + + _load_profile_tools() + + ALL_TOOLS = {cls.name: cls() for cls in get_concrete_subclasses(Tool)} # type: ignore[type-abstract] + ALL_TOOL_SPECS = [tool.spec() for tool in ALL_TOOLS.values()] + + for tool_name, tool in ALL_TOOLS.items(): + logger.info(f"tool registered: {tool_name} - {tool.description}") + + _TOOLS_INITIALIZED = True + + +_initialize_tools() + + +def get_tool_specs(exclusion_list: list[str] = []) -> list[Dict[str, Any]]: + """Get tool specs, optionally excluding some tools.""" + return [spec for spec in ALL_TOOL_SPECS if spec.get("name") not in exclusion_list] + + +# Dispatcher +def _safe_load_obj(args_json: str) -> Dict[str, Any]: + try: + parsed_args = json.loads(args_json or "{}") + return parsed_args if isinstance(parsed_args, dict) else {} + except Exception: + logger.warning("bad args_json=%r", args_json) + return {} + + +async def _dispatch_tool_call(tool_name: str, args: Dict[str, Any], deps: ToolDependencies) -> Dict[str, Any]: + tool = ALL_TOOLS.get(tool_name) + if not tool: + return {"error": f"unknown tool: {tool_name}"} + try: + return await tool(deps, **args) + except asyncio.CancelledError: + logger.info("Tool cancelled: %s", tool_name) + return {"error": "Tool cancelled"} + except Exception as e: + msg = f"{type(e).__name__}: {e}" + logger.exception("Tool error in %s: %s", tool_name, msg) + return {"error": msg} + + +async def dispatch_tool_call(tool_name: str, args_json: str, deps: ToolDependencies) -> Dict[str, Any]: + """Dispatch a tool call by name with JSON args and dependencies.""" + return await _dispatch_tool_call(tool_name, _safe_load_obj(args_json), deps) + + +async def dispatch_tool_call_with_manager(tool_name: str, args_json: str, deps: ToolDependencies, tool_manager: "BackgroundToolManager") -> Dict[str, Any]: + """Dispatch a tool call, injecting a BackgroundToolManager into the args.""" + args = _safe_load_obj(args_json) + args["tool_manager"] = tool_manager + return await _dispatch_tool_call(tool_name, args, deps) diff --git a/src/reachy_mini_receptionist/tools/dance.py b/src/reachy_mini_receptionist/tools/dance.py new file mode 100644 index 0000000000000000000000000000000000000000..f202a1ebc5b4b28a480529dcfffacac34164a9cd --- /dev/null +++ b/src/reachy_mini_receptionist/tools/dance.py @@ -0,0 +1,86 @@ +import logging +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + +# Initialize dance library +try: + from reachy_mini_dances_library.collection.dance import AVAILABLE_MOVES + from reachy_mini_receptionist.dance_emotion_moves import DanceQueueMove + + DANCE_AVAILABLE = True +except ImportError as e: + logger.warning(f"Dance library not available: {e}") + AVAILABLE_MOVES = {} + DANCE_AVAILABLE = False + + +class Dance(Tool): + """Play a named or random dance move once (or repeat). Non-blocking.""" + + name = "dance" + description = "Play a named or random dance move once (or repeat). Non-blocking." + parameters_schema = { + "type": "object", + "properties": { + "move": { + "type": "string", + "description": """Name of the move; use 'random' or omit for random. + Here is a list of the available moves: + simple_nod: A simple, continuous up-and-down nodding motion. + head_tilt_roll: A continuous side-to-side head roll (ear to shoulder). + side_to_side_sway: A smooth, side-to-side sway of the entire head. + dizzy_spin: A circular 'dizzy' head motion combining roll and pitch. + stumble_and_recover: A simulated stumble and recovery with multiple axis movements. Good vibes + interwoven_spirals: A complex spiral motion using three axes at different frequencies. + sharp_side_tilt: A sharp, quick side-to-side tilt using a triangle waveform. + side_peekaboo: A multi-stage peekaboo performance, hiding and peeking to each side. + yeah_nod: An emphatic two-part yeah nod using transient motions. + uh_huh_tilt: A combined roll-and-pitch uh-huh gesture of agreement. + neck_recoil: A quick, transient backward recoil of the neck. + chin_lead: A forward motion led by the chin, combining translation and pitch. + groovy_sway_and_roll: A side-to-side sway combined with a corresponding roll for a groovy effect. + chicken_peck: A sharp, forward, chicken-like pecking motion. + side_glance_flick: A quick glance to the side that holds, then returns. + polyrhythm_combo: A 3-beat sway and a 2-beat nod create a polyrhythmic feel. + grid_snap: A robotic, grid-snapping motion using square waveforms. + pendulum_swing: A simple, smooth pendulum-like swing using a roll motion. + jackson_square: Traces a rectangle via a 5-point path, with sharp twitches on arrival at each checkpoint. + """, + }, + "repeat": { + "type": "integer", + "description": "How many times to repeat the move (default 1).", + }, + }, + "required": [], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Play a named or random dance move once (or repeat). Non-blocking.""" + if not DANCE_AVAILABLE: + return {"error": "Dance system not available"} + + move_name = kwargs.get("move") + repeat = int(kwargs.get("repeat", 1)) + + logger.info("Tool call: dance move=%s repeat=%d", move_name, repeat) + + if not move_name or move_name == "random": + import random + + move_name = random.choice(list(AVAILABLE_MOVES.keys())) + + if move_name not in AVAILABLE_MOVES: + return {"error": f"Unknown dance move '{move_name}'. Available: {list(AVAILABLE_MOVES.keys())}"} + + # Add dance moves to queue + movement_manager = deps.movement_manager + for _ in range(repeat): + dance_move = DanceQueueMove(move_name) + movement_manager.queue_move(dance_move) + + return {"status": "queued", "move": move_name, "repeat": repeat} diff --git a/src/reachy_mini_receptionist/tools/do_nothing.py b/src/reachy_mini_receptionist/tools/do_nothing.py new file mode 100644 index 0000000000000000000000000000000000000000..f8b443dd7e036679effe58a95dc1bb01714f549c --- /dev/null +++ b/src/reachy_mini_receptionist/tools/do_nothing.py @@ -0,0 +1,74 @@ +import logging +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + + +class DoNothing(Tool): + """Choose to do nothing - stay still and silent. Use ONLY when there is no active visitor session. + + HARD GUARD: refuses with an error if called while a visitor flow is in + progress (state != IDLE). The bot was misusing this during + confirmation steps — calling do_nothing 'to stay quiet while waiting + for yes/no' would freeze the conversation forever because the tool + swallows the response cycle. Silence-while-waiting is the realtime + handler's job (VAD), not a tool call. + """ + + name = "do_nothing" + description = ( + "Choose to do nothing — stay still and silent. ONLY valid when " + "state=idle (no active visitor). NEVER call this during a visitor " + "flow to 'wait for a reply' or 'stay quiet for confirmation' — " + "silence between turns happens automatically via VAD. If a " + "visitor session is active, the tool will be rejected." + ) + parameters_schema = { + "type": "object", + "properties": { + "reason": { + "type": "string", + "description": "Optional reason for doing nothing (e.g., 'contemplating existence', 'saving energy', 'being mysterious')", + }, + }, + "required": [], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Do nothing — but refuse if a visitor session is active.""" + reason = kwargs.get("reason", "just chilling") + + # Hard guard: do_nothing during an active visitor flow froze the + # bot into a "staying quiet" loop while ignoring real speech. + sm = getattr(deps, "session_manager", None) + if sm is not None: + try: + current = sm.current_state + state_value = getattr(current, "value", str(current)) + if state_value != "idle": + logger.info( + "[DO_NOTHING BLOCKED] state=%s reason=%r — " + "refusing during active visitor flow", + state_value, reason, + ) + return { + "success": False, + "error": ( + f"do_nothing is not valid while state={state_value!r} " + f"(an active visitor session is in progress). Do NOT " + f"use do_nothing to 'wait for a reply' or 'stay quiet " + f"for confirmation' — silence between turns happens " + f"automatically via VAD. Just stop speaking and the " + f"system will listen for the visitor's response. " + f"Respond normally to whatever the visitor says next." + ), + "blocked_reason": "active_visitor_session", + } + except Exception as e: + logger.debug("do_nothing state check failed: %s", e) + + logger.info("Tool call: do_nothing reason=%s", reason) + return {"status": "doing nothing", "reason": reason} diff --git a/src/reachy_mini_receptionist/tools/get_today_calendar.py b/src/reachy_mini_receptionist/tools/get_today_calendar.py new file mode 100644 index 0000000000000000000000000000000000000000..1f924f1a889d4c5dfd6fbd52b70f6fa2ed8fa3f5 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/get_today_calendar.py @@ -0,0 +1,40 @@ +"""Tool: get_today_calendar + +Returns today's appointment list from the configured iCal feed +(``RECEPTION_ICS_URL``). The LLM uses this to check whether the visitor +in front of the robot has an appointment — and to look up the time, +note, and host email for confirmed visits. + +If no iCal feed is configured, this returns an empty schedule. The bot +then routes any visitor through the walk-in path (``lookup_employee``). +""" +from __future__ import annotations + +from typing import Any, Dict + +from reachy_mini_receptionist.calendar_data import format_for_llm, get_appointments +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +class GetTodayCalendar(Tool): + """Get today's appointment schedule.""" + + name = "get_today_calendar" + description = ( + "Returns today's full appointment schedule. " + "Use this to check if a visitor has an appointment today." + ) + parameters_schema: Dict[str, Any] = { + "type": "object", + "properties": {}, + "required": [], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + appointments = get_appointments() + summary = format_for_llm() + return { + "calendar": appointments, + "summary": summary, + "count": len(appointments), + } diff --git a/src/reachy_mini_receptionist/tools/head_tracking.py b/src/reachy_mini_receptionist/tools/head_tracking.py new file mode 100644 index 0000000000000000000000000000000000000000..43393d367a7d4ab88c42afe996cdcef91989af9d --- /dev/null +++ b/src/reachy_mini_receptionist/tools/head_tracking.py @@ -0,0 +1,31 @@ +import logging +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + + +class HeadTracking(Tool): + """Toggle head tracking state.""" + + name = "head_tracking" + description = "Toggle head tracking state." + parameters_schema = { + "type": "object", + "properties": {"start": {"type": "boolean"}}, + "required": ["start"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Enable or disable head tracking.""" + enable = bool(kwargs.get("start")) + + # Update camera worker head tracking state + if deps.camera_worker is not None: + deps.camera_worker.set_head_tracking_enabled(enable) + + status = "started" if enable else "stopped" + logger.info("Tool call: head_tracking %s", status) + return {"status": f"head tracking {status}"} diff --git a/src/reachy_mini_receptionist/tools/lookup_employee.py b/src/reachy_mini_receptionist/tools/lookup_employee.py new file mode 100644 index 0000000000000000000000000000000000000000..7572ee64d4252f3442b8a48261adcc6c55e60c83 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/lookup_employee.py @@ -0,0 +1,257 @@ +"""Tool: lookup_employee + +Resolves a host name (or alias) to an employee record so the receptionist +can handle "I'm here to see X" walk-ins without that name being on today's +calendar. + +On hit, the result includes the host's email — the LLM can then call +``send_email`` with that address. The conversation controller transitions +the session into APPOINTMENT_MATCHED (with a synthetic walk-in appointment) +on success, or UNKNOWN_EMPLOYEE on miss. +""" +from __future__ import annotations + +import logging +from typing import Any, Dict, List + +from reachy_mini_receptionist import employees +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + +logger = logging.getLogger(__name__) + + +class LookupEmployee(Tool): + """Look up an employee by name or alias and return their contact info.""" + + name = "lookup_employee" + # NOTE (2026-05-21): description softened — confirm-first language + # removed so Gemini doesn't burn an extra turn asking "you're here to + # see X — correct?". Revert to the strict version below when toggling + # VOICE_BACKEND back to "openai" (GPT-4o hallucinated host names, so + # the verbal confirmation step was load-bearing there). + # + # GPT-4o version (re-enable when switching back): + # "Looks up an employee in the company directory by name or alias. " + # "Use this when a visitor says they are here to see a specific person " + # "(e.g. 'I'm here to see Mukul') instead of giving their own name. " + # "MUST be called with confirmed=true, and only after the visitor has " + # "explicitly said YES to your 'You're here to see X — correct?' " + # "question. Calls with confirmed=false will be rejected — the bot " + # "will be instructed to confirm the host first and retry." + description = ( + "Looks up an employee in the company directory by name or alias. " + "Use this when a visitor says they are here to see a specific person " + "(e.g. 'I'm here to see Mukul'). Call it directly as soon as you " + "hear the host's name — do not ask the visitor to verbally confirm " + "the name first. If the directory lookup misses, the response will " + "tell you to offer 2-3 similar-sounding alternatives. Pass " + "confirmed=true on every call." + ) + parameters_schema: Dict[str, Any] = { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": ( + "The name (or short alias) of the employee the visitor " + "asked for, exactly as you heard it." + ), + }, + "confirmed": { + "type": "boolean", + "description": ( + "Always pass true. (Legacy parameter from the GPT-4o " + "backend where the receptionist had to verbally " + "confirm host names before lookup. Currently unused " + "but kept in the schema for clean backend toggling.)" + ), + }, + }, + "required": ["name", "confirmed"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + import asyncio + query = (kwargs.get("name") or "").strip() + confirmed = bool(kwargs.get("confirmed", False)) + if not query: + return {"success": False, "found": False, "error": "Name cannot be empty"} + + # Gemini-backed name disambiguation. Try mapping the transcribed + # host name onto the closest employee-directory entry before the + # confirmed/no-confirmation guards run. Fails open when + # GEMINI_API_KEY is unset. + try: + from reachy_mini_receptionist.name_normalizer import ( + normalize_name, collect_known_names, + ) + normalized = await asyncio.to_thread( + normalize_name, query, collect_known_names() + ) + if normalized and normalized != query: + logger.info("lookup_employee: gemini normalized %r -> %r", query, normalized) + query = normalized + except Exception as e: + logger.debug("lookup_employee: name normalization skipped (%s)", e) + + # ────────────────────────────────────────────────────────────── + # DISABLED (2026-05-21): confirm-first + first-attempt guards. + # Added when GPT-4o was hallucinating host names — forced the LLM + # to verbally confirm before tool execution. Gemini hears names + # cleanly so the extra round-trip is pure latency. Re-enable + # (uncomment) when toggling VOICE_BACKEND back to "openai" for + # A/B comparison. + # ────────────────────────────────────────────────────────────── + # if not confirmed: + # logger.info( + # "[LOOKUP_EMPLOYEE BLOCKED] query=%r called without confirmed=true", + # query, + # ) + # return { + # "success": False, + # "found": False, + # "error": ( + # f"lookup_employee requires confirmed=true. Ask the visitor: " + # f"'You're here to see {query} — correct?', wait for a yes, " + # f"then call lookup_employee again with confirmed=true." + # ), + # "blocked_reason": "no_confirmation", + # } + # + # # First-attempt guard: if the visitor's last utterance is literally + # # the host name, they're stating it for the first time — not + # # confirming. Force the LLM to ask the confirmation question. + # session_manager = getattr(deps, "session_manager", None) + # if session_manager is not None: + # try: + # snap = session_manager.session + # last_user = (snap.last_user_transcript or "").strip().lower() + # except Exception: + # last_user = "" + # query_l = query.lower() + # looks_like_first_attempt = bool(last_user) and ( + # last_user == query_l + # or last_user == f"{query_l}." + # or last_user == f"i'm here to see {query_l}" + # or last_user == f"here to see {query_l}" + # or last_user == f"to see {query_l}" + # ) + # if looks_like_first_attempt: + # logger.warning( + # "[LOOKUP_EMPLOYEE REJECTED] query=%r confirmed=true on first " + # "attempt (transcript=%r)", + # query, last_user, + # ) + # return { + # "success": False, + # "found": False, + # "error": ( + # f"The visitor just said they want {query!r} — they " + # f"haven't confirmed yet. Say: 'You're here to see " + # f"{query} — correct?' and wait for yes." + # ), + # "blocked_reason": "first_attempt_not_confirmed", + # } + + match = employees.lookup_employee(query) + if match is None: + known = self._known_names() + logger.info("[LOOKUP_EMPLOYEE miss] query=%r known=%s", query, known) + return { + "success": True, + "found": False, + "query": query, + "directory": known, + "message": ( + f"No employee matching {query!r} in the directory. " + "Tell the visitor that name isn't on the list. If you " + "may have misheard the name, offer a numbered choice of " + "2-3 similar-sounding directory names plus 'someone else' " + "— do not ask them to spell it. Otherwise offer to take " + "a message." + ), + } + + logger.info( + "[LOOKUP_EMPLOYEE hit] query=%r -> %s <%s>", + query, match.get("name"), match.get("email"), + ) + + # Opportunistic face registration: if the visitor told us their name + # (visitor_name is set in the session) AND we have a current face + # crop AND no face is yet saved under that name, save it now. This + # means the walk-in path ("I'm Krishna, here to see Mukul") also + # populates the face DB so the next visit recognizes them. + try: + self._maybe_save_face(deps) + except Exception as e: + logger.debug("Opportunistic face save failed (non-critical): %s", e) + + host_email = (match.get("email") or "").strip() + host_name = match.get("name") or "" + return { + "success": True, + "found": True, + "employee": { + "name": host_name, + "email": host_email, + "title": match.get("title"), + }, + # Spell it out explicitly so the LLM cannot hallucinate a + # placeholder like 'visiting@methdai.com'. The exact value + # to pass as the `to` parameter of send_email is repeated + # twice in this field for emphasis. + "send_email_to": host_email, + "message": ( + f"Found host: {host_name}. " + f"When you call send_email next, you MUST use exactly " + f"this address as the `to` parameter: {host_email} " + f"Do NOT make up an email address. Copy the value " + f"verbatim from this response." + ), + } + + @staticmethod + def _maybe_save_face(deps: ToolDependencies) -> None: + """Save a face crop under the visitor's spoken name if available. + + No-op when: session has no visitor_name, no face is currently + detected, or the name is already in the face DB. + """ + session_manager = getattr(deps, "session_manager", None) + face_db = getattr(deps, "face_db", None) + worker = getattr(deps, "face_worker", None) or getattr(deps, "vision_processor", None) + if not (session_manager and face_db and worker): + return + + try: + snap = session_manager.session + visitor = (snap.visitor_name or "").strip() + except Exception: + visitor = "" + if not visitor: + return + + # Don't overwrite an existing entry — we'd lose the original crop + # if the new one is bad. + try: + existing = {g.get("name", "").lower() for g in face_db.get_all_guests()} + except Exception: + existing = set() + if visitor.lower() in existing: + return + + face_crop = getattr(worker, "current_encoding", None) + if face_crop is None: + return + + face_db.add_or_update_guest(visitor, face_crop) + try: + worker.rebuild_recognizer() + except Exception as e: + logger.debug("rebuild_recognizer after opportunistic save failed: %s", e) + logger.info("[LOOKUP_EMPLOYEE auto-registered face for %r]", visitor) + + @staticmethod + def _known_names() -> List[str]: + return [e.get("name", "") for e in employees.get_all_employees() if e.get("name")] diff --git a/src/reachy_mini_receptionist/tools/move_head.py b/src/reachy_mini_receptionist/tools/move_head.py new file mode 100644 index 0000000000000000000000000000000000000000..45ec40aa378f8a4a93d0f64cf288db469db6c9a0 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/move_head.py @@ -0,0 +1,79 @@ +import logging +from typing import Any, Dict, Tuple, Literal + +from reachy_mini.utils import create_head_pose +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies +from reachy_mini_receptionist.dance_emotion_moves import GotoQueueMove + + +logger = logging.getLogger(__name__) + +Direction = Literal["left", "right", "up", "down", "front"] + + +class MoveHead(Tool): + """Move head in a given direction.""" + + name = "move_head" + description = "Move your head in a given direction: left, right, up, down or front." + parameters_schema = { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": ["left", "right", "up", "down", "front"], + }, + }, + "required": ["direction"], + } + + # mapping: direction -> args for create_head_pose + DELTAS: Dict[str, Tuple[int, int, int, int, int, int]] = { + "left": (0, 0, 0, 0, 0, 40), + "right": (0, 0, 0, 0, 0, -40), + "up": (0, 0, 0, 0, -30, 0), + "down": (0, 0, 0, 0, 30, 0), + "front": (0, 0, 0, 0, 0, 0), + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Move head in a given direction.""" + direction_raw = kwargs.get("direction") + if not isinstance(direction_raw, str): + return {"error": "direction must be a string"} + direction: Direction = direction_raw # type: ignore[assignment] + logger.info("Tool call: move_head direction=%s", direction) + + deltas = self.DELTAS.get(direction, self.DELTAS["front"]) + target = create_head_pose(*deltas, degrees=True) + + # Use new movement manager + try: + movement_manager = deps.movement_manager + + # Get current state for interpolation + current_head_pose = deps.reachy_mini.get_current_head_pose() + _, current_antennas = deps.reachy_mini.get_current_joint_positions() + + # Create goto move + goto_move = GotoQueueMove( + target_head_pose=target, + start_head_pose=current_head_pose, + target_antennas=(0, 0), # Reset antennas to default + start_antennas=( + current_antennas[0], + current_antennas[1], + ), # Skip body_yaw + target_body_yaw=0, # Reset body yaw + start_body_yaw=current_antennas[0], # body_yaw is first in joint positions + duration=deps.motion_duration_s, + ) + + movement_manager.queue_move(goto_move) + movement_manager.set_moving_state(deps.motion_duration_s) + + return {"status": f"looking {direction}"} + + except Exception as e: + logger.error("move_head failed") + return {"error": f"move_head failed: {type(e).__name__}: {e}"} diff --git a/src/reachy_mini_receptionist/tools/move_head_receptionist.py b/src/reachy_mini_receptionist/tools/move_head_receptionist.py new file mode 100644 index 0000000000000000000000000000000000000000..f1424848dfa7d23fdf20cfa6b51b1b7209d22619 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/move_head_receptionist.py @@ -0,0 +1,104 @@ +"""Tool: move_head (receptionist override) + +Extends the base move_head tool with receptionist-specific named positions: +- "greeting" : slight tilt forward + neutral yaw (attentive look) +- "thinking" : slight head tilt to one side +- "nod" : quick nod sequence +- "front" : neutral / reset position +- All base directions (left, right, up, down) still work. + +Design decision: We override move_head rather than adding a separate tool to keep +the LLM's tool list small. The LLM uses position names like "greeting" naturally. +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any, Dict, Tuple, Literal + +from reachy_mini.utils import create_head_pose +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies +from reachy_mini_receptionist.dance_emotion_moves import GotoQueueMove + +logger = logging.getLogger(__name__) + +Position = Literal["greeting", "thinking", "nod", "front", "left", "right", "up", "down"] + + +class MoveHead(Tool): + """Move Reachy's head to a named position.""" + + name = "move_head" + description = ( + "Move Reachy's head to a named position. " + "Use 'greeting' when a guest arrives, 'thinking' while processing, " + "'nod' to agree or acknowledge, 'front' to reset to neutral. " + "Also supports: left, right, up, down." + ) + parameters_schema: Dict[str, Any] = { + "type": "object", + "properties": { + "position": { + "type": "string", + "enum": ["greeting", "thinking", "nod", "front", "left", "right", "up", "down"], + "description": "The head position or direction.", + } + }, + "required": ["position"], + } + + # (x, y, z, roll, pitch, yaw) deltas in degrees + POSITIONS: Dict[str, Tuple[int, int, int, int, int, int]] = { + "greeting": (0, 0, 0, 0, -15, 0), # slightly forward/up + "thinking": (0, 0, 0, 10, 5, 20), # tilt right, yaw left + "front": (0, 0, 0, 0, 0, 0), # neutral + "left": (0, 0, 0, 0, 0, 40), + "right": (0, 0, 0, 0, 0, -40), + "up": (0, 0, 0, 0, -30, 0), + "down": (0, 0, 0, 0, 30, 0), + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + position = (kwargs.get("position") or "front").lower() + logger.info("move_head position=%s", position) + + if position == "nod": + return await self._do_nod(deps) + + deltas = self.POSITIONS.get(position, self.POSITIONS["front"]) + target = create_head_pose(*deltas, degrees=True) + + try: + movement_manager = deps.movement_manager + current_head_pose = deps.reachy_mini.get_current_head_pose() + _, current_antennas = deps.reachy_mini.get_current_joint_positions() + + goto_move = GotoQueueMove( + target_head_pose=target, + start_head_pose=current_head_pose, + target_antennas=(0, 0), + start_antennas=(current_antennas[0], current_antennas[1]), + target_body_yaw=0, + start_body_yaw=current_antennas[0], + duration=deps.motion_duration_s, + ) + movement_manager.queue_move(goto_move) + movement_manager.set_moving_state(deps.motion_duration_s) + return {"status": f"head moved to '{position}'"} + + except Exception as e: + logger.error("move_head failed: %s", e) + return {"error": f"move_head failed: {type(e).__name__}: {e}"} + + async def _do_nod(self, deps: ToolDependencies) -> Dict[str, Any]: + """Execute a quick 2-step nod sequence.""" + try: + robot = deps.reachy_mini + for pitch in [-20, 5, -10, 0]: + target = create_head_pose(0, 0, 0, 0, pitch, 0, degrees=True) + robot.goto_target(head_pose=target, duration=0.25, wait=False) + await asyncio.sleep(0.3) + return {"status": "nodded"} + except Exception as e: + logger.error("nod failed: %s", e) + return {"error": str(e)} diff --git a/src/reachy_mini_receptionist/tools/notify_ceo.py b/src/reachy_mini_receptionist/tools/notify_ceo.py new file mode 100644 index 0000000000000000000000000000000000000000..ef6fd7e5611f190cfb70b6a54e86c1fd2daedb9a --- /dev/null +++ b/src/reachy_mini_receptionist/tools/notify_ceo.py @@ -0,0 +1,8 @@ +"""notify_ceo.py — DEPRECATED. + +This tool has been replaced by `send_email` (send_email.py). +The file is kept as a stub so existing imports don't crash during transition. +Remove once all references are cleaned up. +""" +# Re-export get_outbox under the old name so main.py can be updated gradually. +from reachy_mini_receptionist.tools.send_email import get_outbox as get_notifications # noqa: F401 diff --git a/src/reachy_mini_receptionist/tools/play_emotion.py b/src/reachy_mini_receptionist/tools/play_emotion.py new file mode 100644 index 0000000000000000000000000000000000000000..7f64ff5fdce7070d59aba19e70f730698a571e8c --- /dev/null +++ b/src/reachy_mini_receptionist/tools/play_emotion.py @@ -0,0 +1,84 @@ +import logging +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + +# Initialize emotion library +try: + from reachy_mini.motion.recorded_move import RecordedMoves + from reachy_mini_receptionist.dance_emotion_moves import EmotionQueueMove + + # Note: huggingface_hub automatically reads HF_TOKEN from environment variables + RECORDED_MOVES = RecordedMoves("pollen-robotics/reachy-mini-emotions-library") + EMOTION_AVAILABLE = True +except ImportError as e: + logger.warning(f"Emotion library not available: {e}") + RECORDED_MOVES = None + EMOTION_AVAILABLE = False + + +def get_available_emotions_and_descriptions() -> str: + """Get formatted list of available emotions with descriptions.""" + if not EMOTION_AVAILABLE: + return "Emotions not available" + + try: + emotion_names = RECORDED_MOVES.list_moves() + output = "Available emotions:\n" + for name in emotion_names: + description = RECORDED_MOVES.get(name).description + output += f" - {name}: {description}\n" + return output + except Exception as e: + return f"Error getting emotions: {e}" + + +class PlayEmotion(Tool): + """Play a pre-recorded emotion.""" + + name = "play_emotion" + description = "Play a pre-recorded emotion" + parameters_schema = { + "type": "object", + "properties": { + "emotion": { + "type": "string", + "description": f"""Name of the emotion to play. + Here is a list of the available emotions: + {get_available_emotions_and_descriptions()} + """, + }, + }, + "required": ["emotion"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Play a pre-recorded emotion.""" + if not EMOTION_AVAILABLE: + return {"error": "Emotion system not available"} + + emotion_name = kwargs.get("emotion") + if not emotion_name: + return {"error": "Emotion name is required"} + + logger.info("Tool call: play_emotion emotion=%s", emotion_name) + + # Check if emotion exists + try: + emotion_names = RECORDED_MOVES.list_moves() + if emotion_name not in emotion_names: + return {"error": f"Unknown emotion '{emotion_name}'. Available: {emotion_names}"} + + # Add emotion to queue + movement_manager = deps.movement_manager + emotion_move = EmotionQueueMove(emotion_name, RECORDED_MOVES) + movement_manager.queue_move(emotion_move) + + return {"status": "queued", "emotion": emotion_name} + + except Exception as e: + logger.exception("Failed to play emotion") + return {"error": f"Failed to play emotion: {e!s}"} diff --git a/src/reachy_mini_receptionist/tools/register_guest.py b/src/reachy_mini_receptionist/tools/register_guest.py new file mode 100644 index 0000000000000000000000000000000000000000..05279766a68d461ecd8f7cc03e24fe9569fe3b33 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/register_guest.py @@ -0,0 +1,510 @@ +"""Tool: register_guest + +Associates the current face crop captured by the camera with the provided name, +and saves it to the file-based guest database. After saving, rebuilds the LBPH +recognizer in the face worker so the guest is immediately recognisable. + +Design decisions: +- If no face is currently detected by the Haar cascade, returns an error so the + LLM can ask the guest to look at the camera again. +- Triggers an antenna wiggle after successful registration for visual feedback. +- Guest capacity is enforced by FaceDatabase (FIFO eviction when full). +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any, Dict, Optional + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + +logger = logging.getLogger(__name__) + + +class RegisterGuest(Tool): + """Register the current visitor's face with their name.""" + + name = "register_guest" + description = ( + "Saves the current visitor's face to the guest database with the provided name. " + "MUST be called with confirmed=true, and only after the visitor has explicitly " + "said YES to your 'I heard X — is that right?' question. Calls with " + "confirmed=false will be rejected — the bot will be instructed to confirm " + "the name first and retry." + ) + parameters_schema: Dict[str, Any] = { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The full name of the guest as they stated it.", + }, + "confirmed": { + "type": "boolean", + "description": ( + "Set to true ONLY after BOTH of these have happened: " + "(1) you said the name back literally — 'I heard " + "— is that right?' using the EXACT name from the " + "transcript (no substitution from calendar/directory); " + "AND (2) the visitor responded with an explicit " + "affirmation: 'yes', 'correct', 'that's right', 'yes " + "that's me'. A simple 'uh-huh', silence, or any other " + "reply is NOT confirmation. If the visitor said no or " + "anything ambiguous, do NOT ask them to spell it — offer " + "a numbered choice of 2-3 phonetically similar names " + "('Did I hear (1) , (2) , or (3) " + "something else?'), wait for a number, confirm the " + "chosen name, then retry. NEVER set confirmed=true on " + "the first attempt — the first attempt is always " + "'I heard — is that right?'." + ), + }, + }, + "required": ["name", "confirmed"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + name = (kwargs.get("name") or "").strip() + confirmed = bool(kwargs.get("confirmed", False)) + if not name: + return {"success": False, "error": "Name cannot be empty"} + + # Gemini-backed name disambiguation. If GEMINI_API_KEY is set, + # ask Gemini whether the transcribed value phonetically matches a + # known calendar/employee name (Lee Win -> Arav, Christina -> Krishna, + # etc). Fails open: returns the original value if Gemini is + # unavailable, so the rest of the flow is unchanged when the key + # is missing. + try: + from reachy_mini_receptionist.name_normalizer import ( + normalize_name, collect_known_names, + ) + normalized = await asyncio.to_thread( + normalize_name, name, collect_known_names() + ) + if normalized and normalized != name: + logger.info("register_guest: gemini normalized %r -> %r", name, normalized) + name = normalized + except Exception as e: + logger.debug("register_guest: name normalization skipped (%s)", e) + + # Strip conversational prefixes the LLM sometimes includes in the + # name field — observed in production: visitor said "It's Henry", + # LLM mistranscribed to "It's Hannah" and called + # register_guest(name="It's Hannah") which would save a face under + # that literal string. The LLM is told to pass JUST the name; this + # is backend cleanup for when the prompt rule isn't followed. + # Order matters — match longest prefix first so "it is" beats "it". + name_lower = name.lower() + for prefix in ("it's ", "it is ", "i'm ", "i am ", "my name is ", "this is ", "that's "): + if name_lower.startswith(prefix): + stripped = name[len(prefix):].strip() + if stripped: + logger.info( + "register_guest: stripped conversational prefix %r from %r -> %r", + prefix, name, stripped, + ) + name = stripped + break + + # Reject obvious filler words / hesitation sounds — observed + # production: visitor says "um, let me think" → STT returns + # "um" → LLM registers visitor as "Um". These are never real + # names; the bot should ask again instead of saving them. + _FILLER_EXACT = { + "um", "uh", "em", "hmm", "huh", "eh", "ah", "oh", + "umm", "uhh", "emm", "hmmm", "uhhh", "ehh", + "yeah", "yes", "no", "okay", "ok", "sure", "right", + "what", "huh?", "sorry", "pardon", + # Polite-phrase leakage — LLM sometimes pulls these from + # its own prior turn ("Hi there!" -> visitor "Thanks"). + "thanks", "thank you", "thank", "please", "hi", "hello", + "hey", "bye", "goodbye", "welcome", + } + if name.lower().strip(".,!?") in _FILLER_EXACT: + logger.warning( + "[REGISTER_GUEST REJECTED] name=%r is a filler/hesitation word", + name, + ) + return { + "success": False, + "error": ( + f"{name!r} is a filler word (um/uh/hmm), not a real " + f"name. Ask the visitor for their name again: " + f"'Sorry, I didn't catch that — what's your name?'" + ), + "blocked_reason": "name_is_filler", + } + + # Common English words the LLM hallucinates as names from its + # own conversation context. Observed: "Community", "Communicator", + # "Reception", "Schedule", "Calendar". Real human names are + # almost never common English nouns in lowercase semantics. + _COMMON_WORD_BLACKLIST = { + "community", "communicator", "reception", "receptionist", + "schedule", "calendar", "appointment", "meeting", "visitor", + "guest", "person", "someone", "anyone", "everybody", "nobody", + "moment", "second", "minute", "hour", "today", "tomorrow", + "yesterday", "morning", "afternoon", "evening", "night", + "name", "names", "host", "hosts", "office", "company", + "team", "lobby", "checkin", "check-in", "checkout", "check-out", + "robot", "system", "model", "machine", "computer", "phone", + "test", "testing", "demo", "example", "sample", "default", + "user", "admin", "operator", "manager", "employee", + } + if name.lower().strip(".,!? ") in _COMMON_WORD_BLACKLIST: + logger.warning( + "[REGISTER_GUEST REJECTED] name=%r is a common word, not a real name", + name, + ) + return { + "success": False, + "error": ( + f"{name!r} is not a real person's name — it sounds " + f"like LLM chatter. Ask the visitor again: " + f"'Sorry, I didn't catch your name — could you " + f"tell me again?'" + ), + "blocked_reason": "name_is_common_word", + } + + # Refuse names that are obviously LLM-generated chatter, not a + # human name. Observed: register_guest(name="I wonder who you + # are", confirmed=true) — the LLM's own thinking text got piped + # into the name field. Real names are short (1-3 tokens), don't + # contain question/filler phrasing. + _NAME_BLACKLIST_SUBSTRINGS = ( + "wonder", "who you are", "who are you", "tell me", "your name", + "what's your", "could you", "please", "sorry", "i don't", "i do not", + "unknown", "visitor", "let me", "hello", "welcome", + ) + name_check = name.lower() + if any(bad in name_check for bad in _NAME_BLACKLIST_SUBSTRINGS): + logger.warning( + "[REGISTER_GUEST REJECTED] name=%r looks like LLM chatter, not a real name", + name, + ) + return { + "success": False, + "error": ( + f"The value {name!r} doesn't look like a person's name. " + f"Ask the visitor: 'What's your name?' and pass JUST the " + f"name they said as the `name` parameter (not a full " + f"sentence). Names are usually 1-3 short words." + ), + "blocked_reason": "name_looks_like_chatter", + } + + # Length sanity — real names are not 6+ words. + if len(name.split()) > 4: + logger.warning( + "[REGISTER_GUEST REJECTED] name=%r has too many words to be a real name", + name, + ) + return { + "success": False, + "error": ( + f"The value {name!r} is too long to be a real name. " + f"Pass just the visitor's name as the `name` parameter." + ), + "blocked_reason": "name_too_long", + } + + session_manager = getattr(deps, "session_manager", None) + + # ────────────────────────────────────────────────────────────── + # DISABLED (2026-05-21): transcript-prefix anti-hallucination guard. + # The check compared the candidate name against the literal + # last_user_transcript text. With Gemini half-cascade Live, the + # ASR transcript is unreliable — observed multilingual garbage + # transcripts ("Vai, né? Isso. E eu vim aqui ver o Ashish.", + # "¿Qué es lo que quieres?") while the model's semantic + # understanding correctly preserved proper nouns and called + # register_guest with the RIGHT names ("Mukul", "David"). The + # guard then rejected those correct names as "hallucinations". + # The confirm-first guard below + the filler/common-word/chatter + # filters above are still in place to defend against actual + # invention. Re-enable when toggling VOICE_BACKEND back to + # "openai" — GPT-4o's transcript IS reliable enough for this check. + # ────────────────────────────────────────────────────────────── + # if session_manager is not None: + # try: + # last_user_for_halluc = (session_manager.session.last_user_transcript or "").strip().lower() + # except Exception: + # last_user_for_halluc = "" + # if last_user_for_halluc and len(name) >= 3: + # name_words = [w.lower() for w in name.split() if len(w) >= 3] + # if name_words and not any( + # w[:3] in last_user_for_halluc for w in name_words + # ): + # logger.warning( + # "[REGISTER_GUEST REJECTED] name=%r doesn't appear in " + # "visitor transcript %r — LLM hallucination", + # name, last_user_for_halluc, + # ) + # return { + # "success": False, + # "error": ( + # f"You called register_guest with name={name!r} " + # f"but the visitor's last transcript was " + # f"{last_user_for_halluc!r}. The name doesn't appear " + # f"in what they said. Do NOT invent names. Ask " + # f"again: 'Sorry, I didn't catch your name — could " + # f"you say it again?'" + # ), + # "blocked_reason": "name_not_in_transcript", + # } + + def _bump_and_check_handoff() -> Optional[Dict[str, Any]]: + """Increment the rejection counter. If we've exceeded 2, return + the operator-handoff response; otherwise return None. + """ + if session_manager is None: + return None + try: + count = session_manager.bump_register_guest_rejection() + except Exception as e: + logger.debug("bump_register_guest_rejection failed: %s", e) + return None + if count >= 3: # 2 actual mishears, on the 3rd attempt give up + logger.warning( + "[REGISTER_GUEST HANDOFF] %d rejections this session — " + "directing bot to operator handoff", + count, + ) + return { + "success": False, + "error": ( + "Too many failed attempts at the visitor's name. " + "Say to the visitor: 'I'm sorry, I'm having trouble " + "catching your name. Let me ask a colleague to help " + "register you — please take a seat for a moment.' " + "Then call do_nothing. An operator will register them " + "from the dashboard." + ), + "blocked_reason": "operator_handoff_after_retries", + } + return None + + if not confirmed: + logger.info( + "[REGISTER_GUEST BLOCKED] name=%r called without confirmed=true", + name, + ) + handoff = _bump_and_check_handoff() + if handoff: + return handoff + return { + "success": False, + "error": ( + f"register_guest requires confirmed=true. The visitor has " + f"NOT yet confirmed the name {name!r}. Say to the visitor: " + f"'I heard {name} — is that right?', wait for them to say " + f"YES (or 'correct', 'that's right'), then call " + f"register_guest again with the same name and " + f"confirmed=true. If they say no or correct you, do NOT " + f"ask them to spell it — offer a numbered choice of 2-3 " + f"similar-sounding names ('Did I hear (1) {name}, (2) " + f", or (3) something else?'), get a number " + f"reply, confirm, then retry." + ), + "blocked_reason": "no_confirmation", + } + + # Narrow backend guard: reject confirmed=true ONLY when the last + # visitor utterance looks like the first attempt (visitor literally + # said the candidate name) — not a confirmation. The LLM has been + # observed registering "Ravi" on the very first turn the visitor + # said "Henry"; this stops that without blocking "no, it's X" + # corrections. + session_manager = getattr(deps, "session_manager", None) + if session_manager is not None: + try: + snap = session_manager.session + last_user = (snap.last_user_transcript or "").strip().lower() + except Exception: + last_user = "" + + # DISABLED (2026-05-21): see longer comment on the first + # transcript-match guard above. Gemini half-cascade emits + # garbage multilingual transcripts even when the model + # heard the name correctly. Re-enable when toggling + # VOICE_BACKEND back to "openai". + # if last_user and len(name) >= 3: + # name_prefix = name.lower()[:3] + # if name_prefix not in last_user: + # logger.warning( + # "[REGISTER_GUEST REJECTED] name=%r doesn't appear in " + # "visitor transcript %r — LLM hallucination", + # name, last_user, + # ) + # return { + # "success": False, + # "error": ( + # f"You called register_guest with name={name!r} " + # f"but the visitor's last transcript was " + # f"{last_user!r} and {name!r} does not appear " + # f"anywhere in it. Do NOT invent names. Ask the " + # f"visitor: 'Sorry, I didn't catch your name — " + # f"could you say it again?'" + # ), + # "blocked_reason": "name_not_in_transcript", + # } + + # Host-intent guard: if the visitor's last utterance was a + # "I want to meet X" / "I'm here to see X" / "Can I talk to X" + # phrase (host intent), the LLM should be calling + # lookup_employee — NOT register_guest. Observed in + # production: visitor said "I want to meet Arjun", LLM + # called both lookup_employee AND register_guest, with the + # register_guest name hallucinated as "Ilya". + _HOST_INTENT_PHRASES = ( + "want to meet", "here to meet", "meet with", + "here to see", "want to see", "to see ", "see him", + "see her", "talk to", "speak to", "speak with", + "looking for", "appointment with", "checking in with", + "here for", "i'm here for", + ) + if any(p in last_user for p in _HOST_INTENT_PHRASES) and name.lower() not in last_user: + logger.warning( + "[REGISTER_GUEST REJECTED] visitor said %r (host intent) but " + "register_guest was called with name=%r — refusing", + last_user, name, + ) + return { + "success": False, + "error": ( + f"The visitor said {last_user!r} — that is a HOST " + f"intent, not the visitor's own name. Do NOT call " + f"register_guest. Instead call lookup_employee with " + f"the host name from the utterance. If you also need " + f"the visitor's own name, ask them: 'And what's your " + f"name?' AFTER you've handled the host." + ), + "blocked_reason": "host_intent_not_visitor_name", + } + + name_l = name.lower() + # If the visitor's last utterance IS the candidate name (or + # contains it as the main content), they were stating their + # name — not confirming. Require the LLM to ask "is that + # right?" and get an explicit yes first. + looks_like_first_attempt = bool(last_user) and ( + last_user == name_l + or last_user == f"i'm {name_l}" + or last_user == f"it's {name_l}" + or last_user == f"my name is {name_l}" + or last_user == f"this is {name_l}" + or last_user == f"{name_l}." + ) + if looks_like_first_attempt: + logger.warning( + "[REGISTER_GUEST REJECTED] name=%r confirmed=true on first " + "attempt — visitor just said the name, hasn't confirmed yet " + "(transcript=%r)", + name, last_user, + ) + handoff = _bump_and_check_handoff() + if handoff: + return handoff + return { + "success": False, + "error": ( + f"Don't register yet. The visitor just told you their " + f"name ({last_user!r}). Say back: 'I heard {name} — " + f"is that right?' and wait for them to say yes before " + f"calling register_guest again." + ), + "blocked_reason": "first_attempt_not_confirmed", + } + + # Get face worker + worker = getattr(deps, "face_worker", None) or getattr(deps, "vision_processor", None) + if worker is None: + return {"success": False, "error": "Face recognition system not available"} + + # current_encoding is now a grayscale face crop (100×100 numpy array) + face_crop = getattr(worker, "current_encoding", None) + if face_crop is None: + return { + "success": False, + "error": "No face detected. Please ask the guest to look directly at the camera.", + } + + # Get the database + face_db = getattr(deps, "face_db", None) + if face_db is None: + return {"success": False, "error": "Guest database not available"} + + # Rename-on-correction: if this session already registered the + # visitor under a different name (e.g. STT first heard "Um" and + # the visitor is now correcting to "David"), delete the old + # face DB entry so we don't keep stale crops. The new name + # below replaces it via add_or_update_guest. + try: + if session_manager is not None: + prev_snap = session_manager.session + prev_name = (prev_snap.visitor_name or "").strip() + if prev_name and prev_name.lower() != name.lower(): + try: + face_db.delete_guest(prev_name) + logger.info( + "register_guest: renaming %r -> %r (deleted old entry)", + prev_name, name, + ) + except Exception as e: + logger.debug("register_guest: delete_guest(%r) failed: %s", prev_name, e) + # Also clear visitor_name on the session so the + # controller's RECOGNIZED transition will fire fresh + # with the new name (otherwise the auto-resolve uses + # the stale name). + try: + session_manager.update(visitor_name=None, recognized_face_name=None) + except Exception as e: + logger.debug("register_guest: session.update clear failed: %s", e) + except Exception as e: + logger.debug("register_guest: rename pre-check failed: %s", e) + + try: + face_db.add_or_update_guest(name, face_crop) + logger.info("Registered guest: %s", name) + + # Rebuild the LBPH recognizer immediately so the worker starts + # recognising this face on the next detection pass. + try: + worker.rebuild_recognizer() + except Exception as e: + logger.warning("Failed to rebuild recognizer after registration: %s", e) + + # Antenna wiggle — excited feedback + await self._wiggle_antennas(deps) + + return { + "success": True, + "message": f"Guest '{name}' has been registered successfully.", + "total_guests": face_db.count(), + } + except Exception as e: + logger.exception("register_guest failed: %s", e) + return {"success": False, "error": str(e)} + + async def _wiggle_antennas(self, deps: ToolDependencies) -> None: + """Wiggle antennas to signal registration excitement.""" + try: + robot = deps.reachy_mini + positions = [ + (45.0, -45.0), + (-45.0, 45.0), + (45.0, -45.0), + (0.0, 0.0), + ] + for right_val, left_val in positions: + robot.goto_target( + right_antenna=right_val, + left_antenna=left_val, + duration=0.2, + wait=False, + ) + await asyncio.sleep(0.25) + except Exception as e: + logger.debug("Antenna wiggle failed (non-critical): %s", e) diff --git a/src/reachy_mini_receptionist/tools/send_email.py b/src/reachy_mini_receptionist/tools/send_email.py new file mode 100644 index 0000000000000000000000000000000000000000..97e0236a543e473802fc57bfd01b4db3c494fae3 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/send_email.py @@ -0,0 +1,321 @@ +"""Tool: send_email + +Sends an email via Resend's transactional email API. Falls back to the +in-memory outbox when ``RESEND_API_KEY`` is unset, so demos / CI keep +working without delivery credentials. + +The outbox is ALWAYS written so the dashboard's Mailbox Out panel reflects +reality regardless of delivery mode. Entries carry ``delivered`` and +``error`` fields so the UI can flag failed sends. + +Environment variables: + RESEND_API_KEY Resend API key (``re_...``). Get one free at + https://resend.com/api-keys. Without this set, the + tool writes to the outbox only and returns + ``success=True, delivered=False`` so the conversation + flow can continue. + RESEND_FROM Sender address. Defaults to ``onboarding@resend.dev`` + (Resend's sandbox sender; this only delivers to the + email address registered on your Resend account). + For production, verify a domain at + https://resend.com/domains and set this to e.g. + ``noreply@methdai.com``. +""" +from __future__ import annotations + +import logging +import os +import time +from typing import Any, Dict, List, Optional + +import httpx + +logger = logging.getLogger(__name__) + +_RESEND_API_URL = "https://api.resend.com/emails" +_HTTP_TIMEOUT_SECONDS = 15.0 + +# --------------------------------------------------------------------------- +# Shared outbox store — read by the dashboard /api/outbox endpoint +# --------------------------------------------------------------------------- +_outbox: List[dict] = [] +_MAX_OUTBOX = 50 + + +def get_outbox() -> List[dict]: + """Return the current outbox (used by dashboard endpoint).""" + return list(_outbox) + + +def clear_outbox() -> int: + """Wipe the in-memory outbox. Used by the dashboard's Demo Reset. + + Returns the count removed. No-op if already empty. + """ + removed = len(_outbox) + _outbox.clear() + return removed + + +def _record_outbox( + to: str, + subject: str, + body: str, + delivered: bool, + error: Optional[str], + provider_message_id: Optional[str] = None, +) -> dict: + entry = { + "timestamp": time.strftime("%H:%M:%S"), + "to": to, + "subject": subject, + "body": body, + "delivered": delivered, + "error": error, + "provider_message_id": provider_message_id, + } + _outbox.append(entry) + if len(_outbox) > _MAX_OUTBOX: + _outbox.pop(0) + return entry + + +# --------------------------------------------------------------------------- +# Tool class +# --------------------------------------------------------------------------- +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies # noqa: E402 + + +class SendEmail(Tool): + """Send an email to the person the guest is visiting.""" + + name = "send_email" + description = ( + "Sends an email notification to the host that their visitor has " + "arrived. PRECONDITION: the visitor's OWN name must be known first " + "— either you heard them say it ('I'm Alex'), or face recognition " + "matched them to a saved guest. If the visitor only named the " + "HOST (e.g. 'I'm here to see Ashish'), you must ask 'And what's " + "your name?' and call register_guest BEFORE calling send_email. " + "Calls without a visitor identity will be rejected. Use the " + "`send_email_to` value from lookup_employee verbatim as the `to` " + "parameter — do not invent an address." + ) + parameters_schema: Dict[str, Any] = { + "type": "object", + "properties": { + "to": { + "type": "string", + "description": "Recipient email address (use the `visiting` field from the calendar).", + }, + "subject": { + "type": "string", + "description": "Email subject line, e.g. 'Guest arrival: Mukul'.", + }, + "body": { + "type": "string", + "description": "Email body, e.g. 'Mukul has arrived for the 11:00 AM internal meeting.'", + }, + }, + "required": ["to", "subject", "body"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + to = (kwargs.get("to") or "").strip() + subject = (kwargs.get("subject") or "").strip() + body = (kwargs.get("body") or "").strip() + + if not to: + return {"success": False, "error": "Recipient address cannot be empty"} + if not subject: + return {"success": False, "error": "Subject cannot be empty"} + if not body: + return {"success": False, "error": "Body cannot be empty"} + + # Reject obvious placeholder/hallucinated addresses. Observed: + # LLM sometimes invents "visiting@methdai.com", "host@...", + # "", etc. instead of using the address from + # lookup_employee's response. + to_lower = to.lower() + _PLACEHOLDER_LOCAL_PARTS = ( + "visiting", "host", "employee", "recipient", "name", + "email", "address", "", "", "placeholder", + "example", "user", "person", "someone", "test", + ) + local = to_lower.split("@", 1)[0] if "@" in to_lower else to_lower + if local in _PLACEHOLDER_LOCAL_PARTS or any( + p in to_lower for p in ("<", ">", "{{", "}}", "example.com") + ): + logger.warning( + "[EMAIL REJECTED → %s] placeholder/hallucinated address", + to, + ) + return { + "success": False, + "error": ( + f"{to!r} is not a real email address. The host's " + f"address comes from lookup_employee's response in " + f"the `employee.email` or `send_email_to` field — " + f"copy that value EXACTLY as the `to` parameter. " + f"Do not invent addresses." + ), + "blocked_reason": "placeholder_email", + } + + session_manager = getattr(deps, "session_manager", None) + + # Hard guard: refuse to notify a host until we have SOME identity + # for the visitor. Accept either an explicit visitor_name (from + # register_guest or the LLM filling it in) OR a recognized face + # match (LBPH already matched a saved guest crop — that's a + # confirmed identity by a different path). Only block when we + # genuinely have no idea who's in front of the robot — otherwise + # we force returning known guests through a redundant name prompt + # which is bad UX. + if session_manager is not None: + try: + snap = session_manager.session + visitor_name = (snap.visitor_name or "").strip() + recognized_face = (snap.recognized_face_name or "").strip() + if not visitor_name and not recognized_face: + logger.info( + "[EMAIL BLOCKED → %s] no visitor identity (name + face both empty); refusing send", + to, + ) + return { + "success": False, + "error": ( + "Visitor's identity is not yet known. Ask the " + "visitor their name, repeat it back to confirm, " + "then call register_guest. Only after the visitor's " + "name is saved should you call send_email." + ), + "delivered": False, + "blocked_reason": "no_visitor_identity", + } + except Exception as e: + logger.debug("send_email visitor-name guard failed: %s", e) + # Fall through — better to send than to silently drop if + # the session API misbehaves. + + # Atomic dedupe: claim the recipient under the SessionManager lock + # before doing any I/O. If two concurrent tool calls fire for the + # same address, exactly one wins the claim — the loser sees the + # claim and returns the duplicate-blocked response without hitting + # Resend. On any failure path below we release the claim so the + # LLM can retry. + claimed = False + if session_manager is not None: + try: + claimed = session_manager.try_claim_email(to) + except Exception as e: + logger.debug("send_email try_claim_email raised: %s", e) + # Fall through and send: better to risk a rare duplicate + # than to silently drop a notification because the claim + # mechanism is misbehaving. + if not claimed and session_manager is not None: + try: + snap = session_manager.session + visitor = snap.visitor_name + except Exception: + visitor = None + logger.info( + "[EMAIL DUPLICATE BLOCKED → %s] Already claimed/sent this session (visitor=%r)", + to, visitor, + ) + return { + "success": True, + "message": ( + f"Email to {to} was already sent or in flight for " + f"this session — skipped duplicate." + ), + "delivered": False, + "duplicate_blocked": True, + } + + def _release_claim() -> None: + if session_manager is not None and claimed: + try: + session_manager.release_email_claim(to) + except Exception as e: + logger.debug("release_email_claim failed: %s", e) + + api_key = os.getenv("RESEND_API_KEY") + if not api_key: + entry = _record_outbox(to, subject, body, delivered=False, error=None) + logger.info( + "[EMAIL DRY-RUN → %s] %s (RESEND_API_KEY unset; outbox only)", + to, subject, + ) + return { + "success": True, + "message": f"Email recorded for {to} (Resend not configured — outbox only)", + "timestamp": entry["timestamp"], + "delivered": False, + } + + sender = os.getenv("RESEND_FROM", "onboarding@resend.dev") + payload = { + "from": sender, + "to": [to], + "subject": subject, + "text": body, + } + + try: + async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT_SECONDS) as client: + resp = await client.post( + _RESEND_API_URL, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json=payload, + ) + except Exception as e: + err = f"{type(e).__name__}: {e}" + entry = _record_outbox(to, subject, body, delivered=False, error=err) + logger.warning("[EMAIL FAILED → %s] %s | %s", to, subject, err) + _release_claim() + return { + "success": False, + "error": err, + "timestamp": entry["timestamp"], + "delivered": False, + } + + if resp.status_code >= 400: + try: + parsed = resp.json() + err_msg = parsed.get("message") or parsed.get("name") or resp.text + except Exception: + err_msg = resp.text + err = f"HTTP {resp.status_code}: {err_msg}" + entry = _record_outbox(to, subject, body, delivered=False, error=err) + logger.warning("[EMAIL FAILED → %s] %s | %s", to, subject, err) + _release_claim() + return { + "success": False, + "error": err, + "timestamp": entry["timestamp"], + "delivered": False, + } + + try: + message_id = resp.json().get("id") + except Exception: + message_id = None + + entry = _record_outbox( + to, subject, body, + delivered=True, error=None, + provider_message_id=message_id, + ) + logger.info("[EMAIL → %s] %s (resend id=%s)", to, subject, message_id) + return { + "success": True, + "message": f"Email sent to {to}", + "timestamp": entry["timestamp"], + "delivered": True, + "provider_message_id": message_id, + } diff --git a/src/reachy_mini_receptionist/tools/stop_dance.py b/src/reachy_mini_receptionist/tools/stop_dance.py new file mode 100644 index 0000000000000000000000000000000000000000..5180a9845def459b9a76715a49b50bd25f83abae --- /dev/null +++ b/src/reachy_mini_receptionist/tools/stop_dance.py @@ -0,0 +1,31 @@ +import logging +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + + +class StopDance(Tool): + """Stop the current dance move.""" + + name = "stop_dance" + description = "Stop the current dance move" + parameters_schema = { + "type": "object", + "properties": { + "dummy": { + "type": "boolean", + "description": "dummy boolean, set it to true", + }, + }, + "required": ["dummy"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Stop the current dance move.""" + logger.info("Tool call: stop_dance") + movement_manager = deps.movement_manager + movement_manager.clear_move_queue() + return {"status": "stopped dance and cleared queue"} diff --git a/src/reachy_mini_receptionist/tools/stop_emotion.py b/src/reachy_mini_receptionist/tools/stop_emotion.py new file mode 100644 index 0000000000000000000000000000000000000000..9baf396d5840b844a110569c56083072956c5239 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/stop_emotion.py @@ -0,0 +1,31 @@ +import logging +from typing import Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies + + +logger = logging.getLogger(__name__) + + +class StopEmotion(Tool): + """Stop the current emotion.""" + + name = "stop_emotion" + description = "Stop the current emotion" + parameters_schema = { + "type": "object", + "properties": { + "dummy": { + "type": "boolean", + "description": "dummy boolean, set it to true", + }, + }, + "required": ["dummy"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Stop the current emotion.""" + logger.info("Tool call: stop_emotion") + movement_manager = deps.movement_manager + movement_manager.clear_move_queue() + return {"status": "stopped emotion and cleared queue"} diff --git a/src/reachy_mini_receptionist/tools/task_cancel.py b/src/reachy_mini_receptionist/tools/task_cancel.py new file mode 100644 index 0000000000000000000000000000000000000000..4bc82f39c253fac740ed4165d73e3d574650c26f --- /dev/null +++ b/src/reachy_mini_receptionist/tools/task_cancel.py @@ -0,0 +1,74 @@ +"""Tool cancel tool - cancel running background tools.""" + +import logging +from typing import TYPE_CHECKING, Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies +from reachy_mini_receptionist.tools.tool_constants import ToolState + + +if TYPE_CHECKING: + from reachy_mini_receptionist.tools.background_tool_manager import BackgroundToolManager + + +logger = logging.getLogger(__name__) + + +class TaskCancel(Tool): + """Cancel a running background tool task.""" + + name = "task_cancel" + description = ( + "Cancel a running background tool task. " + "Use this when the user wants to stop a tool that's running in the background. " + "Requires confirmation before cancelling." + ) + parameters_schema = { + "type": "object", + "properties": { + "tool_id": { + "type": "string", + "description": "The tool ID to cancel", + } + }, + "required": ["tool_id"], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Cancel a background tool.""" + tool_id = kwargs.get("tool_id", "") + tool_manager: BackgroundToolManager | None = kwargs.get("tool_manager") + + if tool_manager is None: + return {"error": "Tool manager is required."} + + logger.info(f"Tool call: tool_cancel tool_id={tool_id}") + + if not tool_id: + return {"error": "Tool ID is required."} + + tool = tool_manager.get_tool(tool_id) + + if not tool: + return {"error": f"Tool {tool_id} not found."} + + # Check if tool is still running + if tool.status != ToolState.RUNNING: + return { + "status": f"{tool.status.value}", + "message": f"Tool '{tool.tool_name}' is not running (status: {tool.status.value}).", + "tool_id": tool_id, + } + + # Cancel the tool + if await tool_manager.cancel_tool(tool_id): + return { + "status": "cancelled", + "message": f"Tool '{tool.tool_name}' has been cancelled.", + "tool_id": tool_id, + "tool_name": tool.tool_name, + } + else: + return { + "error": f"Could not cancel tool {tool_id}. It may have already completed.", + } diff --git a/src/reachy_mini_receptionist/tools/task_status.py b/src/reachy_mini_receptionist/tools/task_status.py new file mode 100644 index 0000000000000000000000000000000000000000..0c304d238adf5010e1fe63a517734327d30d3984 --- /dev/null +++ b/src/reachy_mini_receptionist/tools/task_status.py @@ -0,0 +1,104 @@ +"""Tool status tool - check status of background tools.""" + +import time +import logging +from typing import TYPE_CHECKING, Any, Dict + +from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies +from reachy_mini_receptionist.tools.tool_constants import SystemTool + + +if TYPE_CHECKING: + from reachy_mini_receptionist.tools.background_tool_manager import BackgroundToolManager + + +logger = logging.getLogger(__name__) + + +class TaskStatus(Tool): + """Check status of background tool tasks.""" + + name = "task_status" + description = ( + "Check the status of background tool tasks. " + "Use this when the user asks about running tools or wants to know what's happening in the background." + ) + parameters_schema = { + "type": "object", + "properties": { + "tool_id": { + "type": "string", + "description": "Specific tool ID to check (optional, shows all running tools if omitted)", + }, + }, + "required": [], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Get status of background tools.""" + tool_id: str | None = kwargs.get("tool_id") + tool_manager: BackgroundToolManager | None = kwargs.get("tool_manager") + + if tool_manager is None: + return {"error": "Tool manager is required."} + + logger.info(f"Tool call: tool_status tool_id={tool_id}") + + if tool_id: + tool = tool_manager.get_tool(tool_id) + if not tool: + return {"error": f"Tool {tool_id} not found."} + + result: Dict[str, Any] = { + "tool_id": tool.tool_id, + "name": tool.tool_name, + "status": tool.status.value, + "started_at": tool.started_at, + } + if tool.completed_at: + result["completed_at"] = tool.completed_at + + if tool.progress is not None: + result["progress_percent"] = f"{tool.progress.progress:.0%}" + if tool.progress.message: + result["progress_message"] = tool.progress.message + + if tool.result: + result["result"] = tool.result + if tool.error: + result["error"] = tool.error + + return result + + # Get all running tools + running = tool_manager.get_running_tools() + if not running: + return { + "status": "idle", + "message": "No tools running in the background.", + } + + tools_info = [] + for tool in [tool for tool in running if tool.tool_name not in [system_tool.value for system_tool in SystemTool]]: + elapsed = time.monotonic() - tool.started_at + tool_info: Dict[str, Any] = { + "tool_id": tool.tool_id, + "name": tool.tool_name, + "status": tool.status.value, + "elapsed_seconds": round(elapsed, 1), + } + + # Add progress if tracking + if tool.progress is not None: + tool_info["progress_percent"] = f"{tool.progress.progress:.0%}" + if tool.progress.message: + tool_info["progress_message"] = tool.progress.message + + tools_info.append(tool_info) + + return { + "status": "running", + "count": len(tools_info), + "message": f"{len(tools_info)} tool(s) running in the background.", + "tools": tools_info, + } diff --git a/src/reachy_mini_receptionist/tools/tool_constants.py b/src/reachy_mini_receptionist/tools/tool_constants.py new file mode 100644 index 0000000000000000000000000000000000000000..da163c09140ced0054791ffe674fd39a2a3565ee --- /dev/null +++ b/src/reachy_mini_receptionist/tools/tool_constants.py @@ -0,0 +1,17 @@ +from enum import Enum + + +class ToolState(Enum): + """Status of a background tool.""" + + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class SystemTool(Enum): + """System tools are tools that are used to manage the background tool manager.""" + + TASK_STATUS = "task_status" + TASK_CANCEL = "task_cancel" diff --git a/src/reachy_mini_receptionist/utils.py b/src/reachy_mini_receptionist/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d3cba6abd0cec43bffbf070699b46f9f139d0231 --- /dev/null +++ b/src/reachy_mini_receptionist/utils.py @@ -0,0 +1,148 @@ +import logging +import argparse +import warnings +from typing import Any, Tuple, Optional + +from reachy_mini import ReachyMini +from reachy_mini_receptionist.camera_worker import CameraWorker + + +def parse_args() -> Tuple[argparse.Namespace, list]: # type: ignore + """Parse command line arguments.""" + parser = argparse.ArgumentParser("Reachy Mini Receptionist App") + parser.add_argument( + "--head-tracker", + choices=["yolo", "mediapipe", None], + default=None, + help="Choose head tracker (default: None)", + ) + parser.add_argument("--no-camera", default=False, action="store_true", help="Disable camera usage") + parser.add_argument( + "--local-vision", + default=False, + action="store_true", + help="Use local vision model instead of gpt-realtime vision", + ) + parser.add_argument("--gradio", default=False, action="store_true", help="Open gradio interface") + parser.add_argument("--debug", default=False, action="store_true", help="Enable debug logging") + parser.add_argument( + "--robot-name", + type=str, + default=None, + help="[Optional] Robot name/prefix for Zenoh topics (must match daemon's --robot-name). Only needed for development with multiple robots.", + ) + return parser.parse_known_args() + + +def handle_vision_stuff(args: argparse.Namespace, current_robot: ReachyMini) -> Tuple[CameraWorker | None, Any, Any]: + """Initialize camera, head tracker, camera worker, and vision manager. + + By default, vision is handled by gpt-realtime model when camera tool is used. + If --local-vision flag is used, a local vision model will process images periodically. + """ + camera_worker = None + head_tracker = None + vision_manager = None + + if not args.no_camera: + # Initialize head tracker if specified + if args.head_tracker is not None: + if args.head_tracker == "yolo": + from reachy_mini_receptionist.vision.yolo_head_tracker import HeadTracker + + head_tracker = HeadTracker() + elif args.head_tracker == "mediapipe": + from reachy_mini_toolbox.vision import HeadTracker # type: ignore[no-redef] + + head_tracker = HeadTracker() + + # Initialize camera worker + camera_worker = CameraWorker(current_robot, head_tracker) + + # Initialize vision manager only if local vision is requested + if args.local_vision: + try: + from reachy_mini_receptionist.vision.processors import initialize_vision_manager + + vision_manager = initialize_vision_manager(camera_worker) + except ImportError as e: + raise ImportError( + "To use --local-vision, please install the extra dependencies: pip install '.[local_vision]'", + ) from e + else: + logging.getLogger(__name__).info( + "Using gpt-realtime for vision (default). Use --local-vision for local processing.", + ) + + return camera_worker, head_tracker, vision_manager + + +# Dashboard polling routes that spam the console — suppress them from uvicorn access logs. +_SUPPRESSED_ROUTES = ( + "/api/face_status", + "/api/logs", + "/api/best_face_jpeg", + "/api/guests", + "/api/calendar", + "/api/outbox", + "/video_feed", + "/guest_images/", +) + + +class _SuppressPollingRoutes(logging.Filter): + """Drop uvicorn access-log records for high-frequency dashboard polling endpoints.""" + + def filter(self, record: logging.LogRecord) -> bool: # noqa: A003 + msg = record.getMessage() + return not any(route in msg for route in _SUPPRESSED_ROUTES) + + +def setup_logger(debug: bool) -> logging.Logger: + """Set up the logger.""" + log_level = "DEBUG" if debug else "INFO" + logging.basicConfig( + level=getattr(logging, log_level, logging.INFO), + format="%(asctime)s %(levelname)s %(name)s:%(lineno)d | %(message)s", + ) + logger = logging.getLogger(__name__) + + # Suppress WebRTC warnings + warnings.filterwarnings("ignore", message=".*AVCaptureDeviceTypeExternal.*") + warnings.filterwarnings("ignore", category=UserWarning, module="aiortc") + + # Tame third-party noise (looser in DEBUG) + if log_level == "DEBUG": + logging.getLogger("aiortc").setLevel(logging.INFO) + logging.getLogger("fastrtc").setLevel(logging.INFO) + logging.getLogger("aioice").setLevel(logging.INFO) + logging.getLogger("openai").setLevel(logging.INFO) + logging.getLogger("websockets").setLevel(logging.INFO) + else: + logging.getLogger("aiortc").setLevel(logging.ERROR) + logging.getLogger("fastrtc").setLevel(logging.ERROR) + logging.getLogger("aioice").setLevel(logging.WARNING) + + # Silence high-frequency dashboard polling from uvicorn access logs + logging.getLogger("uvicorn.access").addFilter(_SuppressPollingRoutes()) + + return logger + +def log_connection_troubleshooting(logger: logging.Logger, robot_name: Optional[str]) -> None: + """Log troubleshooting steps for connection issues.""" + logger.error("Troubleshooting steps:") + logger.error(" 1. Verify reachy-mini-daemon is running") + + if robot_name is not None: + logger.error( + f" 2. Daemon must be started with: --robot-name '{robot_name}'" + ) + else: + logger.error( + " 2. If daemon uses --robot-name, add the same flag here: " + "--robot-name " + ) + + logger.error(" 3. For wireless: check network connectivity") + logger.error(" 4. Review daemon logs") + logger.error(" 5. Restart the daemon") diff --git a/src/reachy_mini_receptionist/vision/__init__.py b/src/reachy_mini_receptionist/vision/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..60dc15c497e49cbeb8687afb06a62339c297e2b5 --- /dev/null +++ b/src/reachy_mini_receptionist/vision/__init__.py @@ -0,0 +1 @@ +"""Nothing (for ruff).""" diff --git a/src/reachy_mini_receptionist/vision/processors.py b/src/reachy_mini_receptionist/vision/processors.py new file mode 100644 index 0000000000000000000000000000000000000000..f1d52d7a7c67cd87dc82e5229f0a703e3db57615 --- /dev/null +++ b/src/reachy_mini_receptionist/vision/processors.py @@ -0,0 +1,331 @@ +import os +import time +import base64 +import logging +import threading +from typing import Any, Dict +from dataclasses import dataclass + +import cv2 +import numpy as np +import torch +from numpy.typing import NDArray +from transformers import AutoProcessor, AutoModelForImageTextToText +from huggingface_hub import snapshot_download + +from reachy_mini_receptionist.config import config + + +logger = logging.getLogger(__name__) + + +@dataclass +class VisionConfig: + """Configuration for vision processing.""" + + model_path: str = config.LOCAL_VISION_MODEL + vision_interval: float = 5.0 + max_new_tokens: int = 64 + jpeg_quality: int = 85 + max_retries: int = 3 + retry_delay: float = 1.0 + device_preference: str = "auto" # "auto", "cuda", "cpu" + + +class VisionProcessor: + """Handles SmolVLM2 model loading and inference.""" + + def __init__(self, vision_config: VisionConfig | None = None): + """Initialize the vision processor.""" + self.vision_config = vision_config or VisionConfig() + self.model_path = self.vision_config.model_path + self.device = self._determine_device() + self.processor = None + self.model = None + self._initialized = False + + def _determine_device(self) -> str: + pref = self.vision_config.device_preference + if pref == "cpu": + return "cpu" + if pref == "cuda": + return "cuda" if torch.cuda.is_available() else "cpu" + if pref == "mps": + return "mps" if torch.backends.mps.is_available() else "cpu" + # auto: prefer mps on Apple, then cuda, else cpu + if torch.backends.mps.is_available(): + return "mps" + return "cuda" if torch.cuda.is_available() else "cpu" + + def initialize(self) -> bool: + """Load model and processor onto the selected device.""" + try: + logger.info(f"Loading SmolVLM2 model on {self.device} (HF_HOME={config.HF_HOME})") + self.processor = AutoProcessor.from_pretrained(self.model_path) # type: ignore + + # Select dtype depending on device + if self.device == "cuda": + dtype = torch.bfloat16 + elif self.device == "mps": + dtype = torch.float32 # best for MPS + else: + dtype = torch.float32 + + model_kwargs: Dict[str, Any] = {"dtype": dtype} + + # flash_attention_2 is CUDA-only; skip on MPS/CPU + if self.device == "cuda": + model_kwargs["_attn_implementation"] = "flash_attention_2" + + # Load model weights + self.model = AutoModelForImageTextToText.from_pretrained(self.model_path, **model_kwargs).to(self.device) # type: ignore + + if self.model is not None: + self.model.eval() + self._initialized = True + return True + + except Exception as e: + logger.error(f"Failed to initialize vision model: {e}") + return False + + def process_image( + self, + cv2_image: NDArray[np.uint8], + prompt: str = "Briefly describe what you see in one sentence.", + ) -> str: + """Process CV2 image and return description with retry logic.""" + if not self._initialized or self.processor is None or self.model is None: + return "Vision model not initialized" + + for attempt in range(self.vision_config.max_retries): + try: + # Convert to JPEG bytes + success, jpeg_buffer = cv2.imencode( + ".jpg", + cv2_image, + [cv2.IMWRITE_JPEG_QUALITY, self.vision_config.jpeg_quality], + ) + if not success: + return "Failed to encode image" + + # Convert to base64 + image_base64 = base64.b64encode(jpeg_buffer.tobytes()).decode("utf-8") + + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "url": f"data:image/jpeg;base64,{image_base64}", + }, + {"type": "text", "text": prompt}, + ], + }, + ] + + inputs = self.processor.apply_chat_template( + messages, + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="pt", + ) + + # Move tensors to device WITHOUT forcing dtype (keeps input_ids as torch.long) + inputs = {k: (v.to(self.device) if hasattr(v, "to") else v) for k, v in inputs.items()} + + with torch.no_grad(): + generated_ids = self.model.generate( + **inputs, + do_sample=False, + max_new_tokens=self.vision_config.max_new_tokens, + pad_token_id=self.processor.tokenizer.eos_token_id, + ) + + generated_texts = self.processor.batch_decode( + generated_ids, + skip_special_tokens=True, + ) + + # Extract just the response part + full_text = generated_texts[0] + response = self._extract_response(full_text) + + # Clean up GPU memory if using CUDA + if self.device == "cuda": + torch.cuda.empty_cache() + elif self.device == "mps": + torch.mps.empty_cache() + + return response.replace(chr(10), " ").strip() + + except torch.cuda.OutOfMemoryError as e: + logger.error(f"CUDA OOM on attempt {attempt + 1}: {e}") + if self.device == "cuda": + torch.cuda.empty_cache() + if attempt < self.vision_config.max_retries - 1: + time.sleep(self.vision_config.retry_delay * (attempt + 1)) + else: + return "GPU out of memory - vision processing failed" + + except Exception as e: + logger.error(f"Vision processing failed (attempt {attempt + 1}): {e}") + if attempt < self.vision_config.max_retries - 1: + time.sleep(self.vision_config.retry_delay) + else: + return f"Vision processing error after {self.vision_config.max_retries} attempts" + + def _extract_response(self, full_text: str) -> str: + """Extract the assistant's response from the full generated text.""" + # Handle different response formats + markers = ["assistant\n", "Assistant:", "Response:", "\n\n"] + + for marker in markers: + if marker in full_text: + response = full_text.split(marker)[-1].strip() + if response: # Ensure we got a meaningful response + return response + + # Fallback: return the full text cleaned up + return full_text.strip() + + def get_model_info(self) -> Dict[str, Any]: + """Get information about the loaded model.""" + return { + "initialized": self._initialized, + "device": self.device, + "model_path": self.model_path, + "cuda_available": torch.cuda.is_available(), + "gpu_memory": torch.cuda.get_device_properties(0).total_memory // (1024**3) + if torch.cuda.is_available() + else "N/A", + } + + +class VisionManager: + """Manages periodic vision processing and scene understanding.""" + + def __init__(self, camera: Any, vision_config: VisionConfig | None = None): + """Initialize vision manager with camera and configuration.""" + self.camera = camera + self.vision_config = vision_config or VisionConfig() + self.vision_interval = self.vision_config.vision_interval + self.processor = VisionProcessor(self.vision_config) + + self._last_processed_time = 0.0 + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + + # Initialize processor + if not self.processor.initialize(): + logger.error("Failed to initialize vision processor") + raise RuntimeError("Vision processor initialization failed") + + def start(self) -> None: + """Start the vision processing loop in a thread.""" + self._stop_event.clear() + self._thread = threading.Thread(target=self._working_loop, daemon=True) + self._thread.start() + logger.info("Local vision processing started") + + def stop(self) -> None: + """Stop the vision processing loop.""" + self._stop_event.set() + if self._thread is not None: + self._thread.join() + logger.info("Local vision processing stopped") + + def _working_loop(self) -> None: + """Vision processing loop (runs in separate thread).""" + while not self._stop_event.is_set(): + try: + current_time = time.time() + + if current_time - self._last_processed_time >= self.vision_interval: + frame = self.camera.get_latest_frame() + if frame is not None: + description = self.processor.process_image( + frame, + "Briefly describe what you see in one sentence.", + ) + + # Only update if we got a valid response + if description and not description.startswith(("Vision", "Failed", "Error")): + self._last_processed_time = current_time + logger.debug(f"Vision update: {description}") + else: + logger.warning(f"Invalid vision response: {description}") + + time.sleep(1.0) # Check every second + + except Exception: + logger.exception("Vision processing loop error") + time.sleep(5.0) # Longer sleep on error + + logger.info("Vision loop finished") + + def get_status(self) -> Dict[str, Any]: + """Get comprehensive status information.""" + return { + "last_processed": self._last_processed_time, + "processor_info": self.processor.get_model_info(), + "config": { + "interval": self.vision_interval, + }, + } + + +def initialize_vision_manager(camera_worker: Any) -> VisionManager | None: + """Initialize vision manager with model download and configuration. + + Args: + camera_worker: CameraWorker instance for frame capture + Returns: + VisionManager instance or None if initialization fails + + """ + try: + model_id = config.LOCAL_VISION_MODEL + cache_dir = os.path.expanduser(config.HF_HOME) + + # Prepare cache directory + os.makedirs(cache_dir, exist_ok=True) + os.environ["HF_HOME"] = cache_dir + logger.info("HF_HOME set to %s", cache_dir) + + # Download model to cache + logger.info(f"Downloading vision model {model_id} to cache...") + snapshot_download( + repo_id=model_id, + repo_type="model", + cache_dir=cache_dir, + ) + logger.info(f"Model {model_id} downloaded to {cache_dir}") + + # Configure vision processing + vision_config = VisionConfig( + model_path=model_id, + vision_interval=5.0, + max_new_tokens=64, + jpeg_quality=85, + max_retries=3, + retry_delay=1.0, + device_preference="auto", + ) + + # Initialize vision manager + vision_manager = VisionManager(camera_worker, vision_config) + + # Log device info + device_info = vision_manager.processor.get_model_info() + logger.info( + f"Vision processing enabled: {device_info.get('model_path')} on {device_info.get('device')}", + ) + + return vision_manager + + except Exception as e: + logger.error(f"Failed to initialize vision manager: {e}") + return None diff --git a/src/reachy_mini_receptionist/vision/yolo_head_tracker.py b/src/reachy_mini_receptionist/vision/yolo_head_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..3619bc7e89a05a21f554c36c7968bf2c446a33f0 --- /dev/null +++ b/src/reachy_mini_receptionist/vision/yolo_head_tracker.py @@ -0,0 +1,148 @@ +from __future__ import annotations +import logging +from typing import Tuple + +import numpy as np +from numpy.typing import NDArray + + +try: + from supervision import Detections + from ultralytics import YOLO # type: ignore +except ImportError as e: + raise ImportError( + "To use YOLO head tracker, please install the extra dependencies: pip install '.[yolo_vision]'", + ) from e +from huggingface_hub import hf_hub_download + + +logger = logging.getLogger(__name__) + + +class HeadTracker: + """Lightweight head tracker using YOLO for face detection.""" + + def __init__( + self, + model_repo: str = "AdamCodd/YOLOv11n-face-detection", + model_filename: str = "model.pt", + confidence_threshold: float = 0.3, + device: str = "cpu", + ) -> None: + """Initialize YOLO-based head tracker. + + Args: + model_repo: HuggingFace model repository + model_filename: Model file name + confidence_threshold: Minimum confidence for face detection + device: Device to run inference on ('cpu' or 'cuda') + + """ + self.confidence_threshold = confidence_threshold + + try: + # Download and load YOLO model + model_path = hf_hub_download(repo_id=model_repo, filename=model_filename) + self.model = YOLO(model_path).to(device) + logger.info(f"YOLO face detection model loaded from {model_repo}") + except Exception as e: + logger.error(f"Failed to load YOLO model: {e}") + raise + + def _select_best_face(self, detections: Detections) -> int | None: + """Select the best face based on confidence and area (largest face with highest confidence). + + Args: + detections: Supervision detections object + + Returns: + Index of best face or None if no valid faces + + """ + if detections.xyxy.shape[0] == 0: + return None + + # Check if confidence is available + if detections.confidence is None: + return None + + # Filter by confidence threshold + valid_mask = detections.confidence >= self.confidence_threshold + if not np.any(valid_mask): + return None + + valid_indices = np.where(valid_mask)[0] + + # Calculate areas for valid detections + boxes = detections.xyxy[valid_indices] + areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) + + # Combine confidence and area (weighted towards larger faces) + confidences = detections.confidence[valid_indices] + scores = confidences * 0.7 + (areas / np.max(areas)) * 0.3 + + # Return index of best face + best_idx = valid_indices[np.argmax(scores)] + return int(best_idx) + + def _bbox_to_mp_coords(self, bbox: NDArray[np.float32], w: int, h: int) -> NDArray[np.float32]: + """Convert bounding box center to MediaPipe-style coordinates [-1, 1]. + + Args: + bbox: Bounding box [x1, y1, x2, y2] + w: Image width + h: Image height + + Returns: + Center point in [-1, 1] coordinates + + """ + center_x = (bbox[0] + bbox[2]) / 2.0 + center_y = (bbox[1] + bbox[3]) / 2.0 + + # Normalize to [0, 1] then to [-1, 1] + norm_x = (center_x / w) * 2.0 - 1.0 + norm_y = (center_y / h) * 2.0 - 1.0 + + return np.array([norm_x, norm_y], dtype=np.float32) + + def get_head_position(self, img: NDArray[np.uint8]) -> Tuple[NDArray[np.float32] | None, float | None]: + """Get head position from face detection. + + Args: + img: Input image + + Returns: + Tuple of (eye_center [-1,1], roll_angle) + + """ + h, w = img.shape[:2] + + try: + # Run YOLO inference + results = self.model(img, verbose=False) + detections = Detections.from_ultralytics(results[0]) + + # Select best face + face_idx = self._select_best_face(detections) + if face_idx is None: + logger.debug("No face detected above confidence threshold") + return None, None + + bbox = detections.xyxy[face_idx] + + if detections.confidence is not None: + confidence = detections.confidence[face_idx] + logger.debug(f"Face detected with confidence: {confidence:.2f}") + + # Get face center in [-1, 1] coordinates + face_center = self._bbox_to_mp_coords(bbox, w, h) + + # Roll is 0 since we don't have keypoints for precise angle estimation + roll = 0.0 + + return face_center, roll + + except Exception as e: + logger.error(f"Error in head position detection: {e}") + return None, None diff --git a/src/reachy_mini_receptionist/visitor_log.py b/src/reachy_mini_receptionist/visitor_log.py new file mode 100644 index 0000000000000000000000000000000000000000..16e239274b2fe514f35a2232f34392d39e3a4492 --- /dev/null +++ b/src/reachy_mini_receptionist/visitor_log.py @@ -0,0 +1,298 @@ +"""SQLite-backed visitor log. + +Records one row per completed visitor session: who arrived, who they came +to see, whether the host was emailed, and what state the session ended in. +Written by ConversationController on session reset; read by the dashboard +via ``/api/visitor_log``. + +Single SQLite file under the app's instance directory (default: cwd next +to ``guests/``). One writer at a time (the receptionist sees one visitor +at a time), so a single ``threading.Lock`` plus per-call connections +gives us safe concurrent reads without holding the DB open. +""" +from __future__ import annotations + +import logging +import sqlite3 +import threading +import time +from datetime import datetime +from pathlib import Path +from typing import Any, Optional + +from reachy_mini_receptionist.receptionist_state import ReceptionState +from reachy_mini_receptionist.session_manager import VisitorSession + +logger = logging.getLogger(__name__) + + +_SCHEMA_VERSION = 1 + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS visits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at TEXT NOT NULL, + ended_at TEXT NOT NULL, + started_at_epoch REAL NOT NULL, + visitor_name TEXT, + recognized_face_name TEXT, + employee_name TEXT, + matched_appointment_time TEXT, + matched_appointment_note TEXT, + email_sent_to TEXT, + final_state TEXT NOT NULL, + error_message TEXT +); + +CREATE INDEX IF NOT EXISTS idx_visits_started_at_epoch + ON visits (started_at_epoch DESC); + +CREATE INDEX IF NOT EXISTS idx_visits_visitor_name + ON visits (visitor_name); + +CREATE INDEX IF NOT EXISTS idx_visits_final_state + ON visits (final_state); +""" + + +class VisitorLog: + """Append-only log of completed visits.""" + + def __init__(self, db_path: str | Path) -> None: + self._db_path = Path(db_path) + self._db_path.parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.Lock() + self._init_schema() + logger.info("VisitorLog initialised at %s", self._db_path) + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + def _connect(self) -> sqlite3.Connection: + # check_same_thread=False because we lock around every call ourselves. + conn = sqlite3.connect(self._db_path, check_same_thread=False, timeout=5.0) + conn.row_factory = sqlite3.Row + # synchronous=NORMAL is the recommended pairing for WAL (durable but + # faster than FULL — we accept losing the last few seconds on a hard + # crash, which is fine for visitor logs). + # foreign_keys=ON is per-connection; we have none yet but it's a + # good default for future joins. + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA foreign_keys=ON") + return conn + + def _init_schema(self) -> None: + with self._lock: + conn = self._connect() + try: + # WAL mode = better concurrent reads while writes are in + # flight, and improves crash recovery. journal_mode is + # persistent in the DB file, so set once at init. + conn.execute("PRAGMA journal_mode=WAL") + + conn.executescript(_SCHEMA) + + # Stamp the schema version so future migrations have an + # anchor. Single-table for now so no migration logic yet. + conn.execute( + "INSERT OR REPLACE INTO schema_meta (key, value) VALUES (?, ?)", + ("schema_version", str(_SCHEMA_VERSION)), + ) + conn.commit() + finally: + conn.close() + + def schema_version(self) -> int: + """Return the on-disk schema version (used by future migrations).""" + with self._lock: + conn = self._connect() + try: + row = conn.execute( + "SELECT value FROM schema_meta WHERE key = ?", + ("schema_version",), + ).fetchone() + finally: + conn.close() + return int(row["value"]) if row else 0 + + # ------------------------------------------------------------------ + # Write + # ------------------------------------------------------------------ + + def record_visit( + self, + snapshot: VisitorSession, + final_state: ReceptionState, + ) -> Optional[int]: + """Persist a completed visit. Returns the new row id, or None if skipped. + + Skips when the snapshot has nothing meaningful to log (no visitor + name, no recognized face, no employee assigned) — that means the + session was just a transient face flicker, not a real visit. + """ + if not (snapshot.visitor_name or snapshot.recognized_face_name or snapshot.employee_name): + logger.debug("VisitorLog: skipping empty visit (final_state=%s)", final_state.value) + return None + + appt = snapshot.matched_appointment or {} + row = ( + datetime.fromtimestamp(snapshot.started_at).isoformat(timespec="seconds"), + datetime.fromtimestamp(time.time()).isoformat(timespec="seconds"), + float(snapshot.started_at), + snapshot.visitor_name, + snapshot.recognized_face_name, + snapshot.employee_name, + appt.get("time"), + appt.get("note"), + snapshot.email_sent_to, + final_state.value, + snapshot.error_message, + ) + + with self._lock: + conn = self._connect() + try: + cur = conn.execute( + """ + INSERT INTO visits ( + started_at, ended_at, started_at_epoch, + visitor_name, recognized_face_name, employee_name, + matched_appointment_time, matched_appointment_note, + email_sent_to, final_state, error_message + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + row, + ) + conn.commit() + visit_id = cur.lastrowid + finally: + conn.close() + + logger.info( + "VisitorLog: recorded visit id=%s visitor=%r employee=%r final_state=%s", + visit_id, snapshot.visitor_name, snapshot.employee_name, final_state.value, + ) + return visit_id + + # ------------------------------------------------------------------ + # Read + # ------------------------------------------------------------------ + + def list_visits(self, limit: int = 100) -> list[dict[str, Any]]: + """Return the most-recent ``limit`` visits, newest first.""" + limit = max(1, min(int(limit), 1000)) + with self._lock: + conn = self._connect() + try: + rows = conn.execute( + """ + SELECT * FROM visits + ORDER BY started_at_epoch DESC + LIMIT ? + """, + (limit,), + ).fetchall() + finally: + conn.close() + return [dict(r) for r in rows] + + def count_today(self) -> int: + """Number of visits started today (server local time).""" + today_start = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + epoch = today_start.timestamp() + with self._lock: + conn = self._connect() + try: + row = conn.execute( + "SELECT COUNT(*) FROM visits WHERE started_at_epoch >= ?", + (epoch,), + ).fetchone() + finally: + conn.close() + return int(row[0]) if row else 0 + + def count_emails_delivered_today(self) -> int: + """Visits today where an email was actually sent to a host.""" + today_start = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + epoch = today_start.timestamp() + with self._lock: + conn = self._connect() + try: + row = conn.execute( + """ + SELECT COUNT(*) FROM visits + WHERE started_at_epoch >= ? + AND email_sent_to IS NOT NULL AND email_sent_to != '' + """, + (epoch,), + ).fetchone() + finally: + conn.close() + return int(row[0]) if row else 0 + + def last_visit(self) -> Optional[dict[str, Any]]: + """Return the most-recent visit row, or None.""" + with self._lock: + conn = self._connect() + try: + row = conn.execute( + """ + SELECT * FROM visits + ORDER BY started_at_epoch DESC + LIMIT 1 + """, + ).fetchone() + finally: + conn.close() + return dict(row) if row else None + + def wipe_all(self) -> int: + """Delete every visit row. Used by the dashboard's Demo Reset. + + Returns the number of rows removed. Schema + WAL state are left + intact so the next write reuses the same file. + """ + with self._lock: + conn = self._connect() + try: + cur = conn.execute("DELETE FROM visits") + conn.commit() + removed = cur.rowcount or 0 + finally: + conn.close() + logger.info("VisitorLog: wiped %d row(s)", removed) + return removed + + def cleanup_older_than(self, max_age_days: float) -> int: + """Delete visit rows whose ``started_at_epoch`` is older than the cutoff. + + Returns the number of rows removed. Pass 0 or negative to disable + cleanup. Runs a single DELETE under the same lock used for writes; + the WAL keeps concurrent readers unblocked. + """ + if max_age_days <= 0: + return 0 + cutoff_epoch = time.time() - (max_age_days * 86400.0) + with self._lock: + conn = self._connect() + try: + cur = conn.execute( + "DELETE FROM visits WHERE started_at_epoch < ?", + (cutoff_epoch,), + ) + conn.commit() + removed = cur.rowcount or 0 + finally: + conn.close() + if removed: + logger.info( + "Visitor log: removed %d row(s) older than %.1f days", + removed, max_age_days, + ) + return removed diff --git a/style.css b/style.css new file mode 100644 index 0000000000000000000000000000000000000000..e137a1bd83c2e99504f4d3db45c6c2de77d041c3 --- /dev/null +++ b/style.css @@ -0,0 +1,437 @@ +:root { + --bg: #060c1d; + --panel: #0c172b; + --glass: rgba(17, 27, 48, 0.7); + --card: rgba(255, 255, 255, 0.04); + --accent: #7af5c4; + --accent-2: #f6c452; + --text: #e8edf7; + --muted: #9fb3ce; + --border: rgba(255, 255, 255, 0.08); + --shadow: 0 25px 70px rgba(0, 0, 0, 0.45); + font-family: "Space Grotesk", "Manrope", system-ui, -apple-system, sans-serif; +} + +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} + +body { + background: radial-gradient(circle at 20% 20%, rgba(122, 245, 196, 0.12), transparent 30%), + radial-gradient(circle at 80% 0%, rgba(246, 196, 82, 0.14), transparent 32%), + radial-gradient(circle at 50% 70%, rgba(124, 142, 255, 0.1), transparent 30%), + var(--bg); + color: var(--text); + min-height: 100vh; + line-height: 1.6; + padding-bottom: 3rem; +} + +a { + color: inherit; + text-decoration: none; +} + +.hero { + padding: 3.5rem clamp(1.5rem, 3vw, 3rem) 2.5rem; + position: relative; + overflow: hidden; +} + +.hero::after { + content: ""; + position: absolute; + inset: 0; + background: linear-gradient(120deg, rgba(122, 245, 196, 0.12), rgba(246, 196, 82, 0.08), transparent); + pointer-events: none; +} + +.topline { + display: flex; + align-items: center; + justify-content: space-between; + max-width: 1200px; + margin: 0 auto 2rem; + position: relative; + z-index: 2; +} + +.brand { + display: flex; + align-items: center; + gap: 0.5rem; + font-weight: 700; + letter-spacing: 0.5px; + color: var(--text); +} + +.logo { + display: inline-flex; + align-items: center; + justify-content: center; + width: 2.2rem; + height: 2.2rem; + border-radius: 10px; + background: linear-gradient(145deg, rgba(122, 245, 196, 0.15), rgba(124, 142, 255, 0.15)); + box-shadow: 0 10px 30px rgba(0, 0, 0, 0.25); +} + +.brand-name { + font-size: 1.1rem; +} + +.pill { + background: rgba(255, 255, 255, 0.06); + border: 1px solid var(--border); + padding: 0.6rem 1rem; + border-radius: 999px; + color: var(--muted); + font-size: 0.9rem; + box-shadow: 0 12px 30px rgba(0, 0, 0, 0.2); +} + +.hero-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); + gap: clamp(1.5rem, 2.5vw, 2.5rem); + max-width: 1200px; + margin: 0 auto; + position: relative; + z-index: 2; + align-items: center; +} + +.hero-copy h1 { + font-size: clamp(2.6rem, 4vw, 3.6rem); + margin-bottom: 1rem; + line-height: 1.1; + letter-spacing: -0.5px; +} + +.live-demo-badge { + display: inline-flex; + align-items: center; + margin-left: 0.65rem; + padding: 0.2rem 0.55rem; + font-size: 0.34em; + line-height: 1; + letter-spacing: 0.02em; + font-weight: 600; + border-radius: 999px; + border: 1px solid var(--border); + color: var(--muted); + background: rgba(255, 255, 255, 0.04); + vertical-align: middle; +} + +.live-demo-badge:hover { + color: var(--text); + border-color: rgba(255, 255, 255, 0.22); + background: rgba(255, 255, 255, 0.07); +} + +.eyebrow { + display: inline-flex; + align-items: center; + gap: 0.5rem; + text-transform: uppercase; + letter-spacing: 1px; + font-size: 0.8rem; + color: var(--muted); + margin-bottom: 0.75rem; +} + +.eyebrow::before { + content: ""; + display: inline-block; + width: 24px; + height: 2px; + background: linear-gradient(90deg, var(--accent), var(--accent-2)); + border-radius: 999px; +} + +.lede { + font-size: 1.1rem; + color: var(--muted); + max-width: 620px; +} + +.hero-actions { + display: flex; + gap: 1rem; + align-items: center; + margin: 1.6rem 0 1.2rem; + flex-wrap: wrap; +} + +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.6rem; + padding: 0.85rem 1.4rem; + border-radius: 12px; + font-weight: 700; + border: 1px solid transparent; + cursor: pointer; + transition: transform 0.2s ease, box-shadow 0.2s ease, background 0.2s ease, border-color 0.2s ease; +} + +.btn.primary { + background: linear-gradient(135deg, #7af5c4, #7c8eff); + color: #0a0f1f; + box-shadow: 0 15px 30px rgba(122, 245, 196, 0.25); +} + +.btn.primary:hover { + transform: translateY(-2px); + box-shadow: 0 25px 45px rgba(122, 245, 196, 0.35); +} + +.btn.ghost { + background: rgba(255, 255, 255, 0.05); + border-color: var(--border); + color: var(--text); +} + +.btn.ghost:hover { + border-color: rgba(255, 255, 255, 0.3); + transform: translateY(-2px); +} + +.btn.wide { + width: 100%; + justify-content: center; +} + +.hero-badges { + display: flex; + flex-wrap: wrap; + gap: 0.6rem; + color: var(--muted); + font-size: 0.9rem; +} + +.hero-badges span { + padding: 0.5rem 0.8rem; + border-radius: 10px; + border: 1px solid var(--border); + background: rgba(255, 255, 255, 0.04); +} + +.hero-visual .glass-card { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 18px; + padding: 1.2rem; + box-shadow: var(--shadow); + backdrop-filter: blur(10px); +} + +.hero-gif { + width: 100%; + display: block; + border-radius: 14px; + border: 1px solid var(--border); + box-shadow: 0 12px 35px rgba(0, 0, 0, 0.35); +} + +.caption { + margin-top: 0.75rem; + color: var(--muted); + font-size: 0.95rem; +} + +.video-embed-wrapper { + margin-top: 0.9rem; + position: relative; + width: 100%; + aspect-ratio: 16 / 9; + border-radius: 12px; + overflow: hidden; + border: 1px solid var(--border); + background: rgba(0, 0, 0, 0.35); +} + +.video-embed-wrapper iframe { + width: 100%; + height: 100%; + display: block; + border: 0; +} + +.video-link { + margin-top: 0.55rem; + font-size: 0.85rem; + color: var(--muted); +} + +.video-link a { + text-decoration: underline; + text-underline-offset: 2px; +} + +.section { + max-width: 1200px; + margin: 0 auto; + padding: clamp(2rem, 4vw, 3.5rem) clamp(1.5rem, 3vw, 3rem); +} + +.section-header { + text-align: center; + max-width: 780px; + margin: 0 auto 2rem; +} + +.section-header h2 { + font-size: clamp(2rem, 3vw, 2.6rem); + margin-bottom: 0.5rem; +} + +.intro { + color: var(--muted); + font-size: 1.05rem; +} + +.feature-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 1rem; +} + +.feature-card { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 16px; + padding: 1.25rem; + box-shadow: 0 10px 30px rgba(0, 0, 0, 0.2); + transition: transform 0.2s ease, border-color 0.2s ease, box-shadow 0.2s ease; +} + +.feature-card:hover { + transform: translateY(-4px); + border-color: rgba(122, 245, 196, 0.3); + box-shadow: 0 18px 40px rgba(0, 0, 0, 0.3); +} + +.feature-card .icon { + width: 48px; + height: 48px; + border-radius: 12px; + display: grid; + place-items: center; + background: rgba(122, 245, 196, 0.14); + margin-bottom: 0.8rem; + font-size: 1.4rem; +} + +.feature-card h3 { + margin-bottom: 0.35rem; +} + +.feature-card p { + color: var(--muted); +} + +.story { + padding-top: 1rem; +} + +.story-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); + gap: 1rem; +} + +.story-card { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--border); + border-radius: 18px; + padding: 1.5rem; + box-shadow: var(--shadow); +} + +.story-card.secondary { + background: linear-gradient(145deg, rgba(124, 142, 255, 0.08), rgba(122, 245, 196, 0.06)); +} + +.story-card h3 { + margin-bottom: 0.8rem; +} + +.story-list { + list-style: none; + display: grid; + gap: 0.7rem; + color: var(--muted); + font-size: 0.98rem; +} + +.story-list li { + display: flex; + gap: 0.7rem; + align-items: flex-start; +} + +.story-text { + color: var(--muted); + line-height: 1.7; + margin-bottom: 1rem; +} + +.chips { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; +} + +.chip { + padding: 0.45rem 0.8rem; + border-radius: 12px; + background: rgba(0, 0, 0, 0.2); + border: 1px solid var(--border); + color: var(--text); + font-size: 0.9rem; +} + +.footer { + text-align: center; + color: var(--muted); + padding: 2rem 1.5rem 0; +} + +.footer a { + color: var(--text); + border-bottom: 1px solid transparent; +} + +.footer a:hover { + border-color: rgba(255, 255, 255, 0.5); +} + +@media (max-width: 768px) { + .hero { + padding-top: 2.5rem; + } + + .topline { + flex-direction: column; + gap: 0.8rem; + align-items: flex-start; + } + + .hero-actions { + width: 100%; + } + + .btn { + width: 100%; + justify-content: center; + } + + .hero-badges { + gap: 0.4rem; + } +} diff --git a/tests/audio/test_head_wobbler.py b/tests/audio/test_head_wobbler.py new file mode 100644 index 0000000000000000000000000000000000000000..07b4650ec8dc349289224ba05eb44026785fbf0d --- /dev/null +++ b/tests/audio/test_head_wobbler.py @@ -0,0 +1,110 @@ +"""Regression tests for the audio-driven head wobble behaviour.""" + +import math +import time +import base64 +import threading +from typing import Any, List, Tuple +from collections.abc import Callable + +import numpy as np + +from reachy_mini_receptionist.audio.head_wobbler import HeadWobbler + + +def _make_audio_chunk(duration_s: float = 0.3, frequency_hz: float = 220.0) -> str: + """Generate a base64-encoded mono PCM16 sine wave.""" + sample_rate = 24000 + sample_count = int(sample_rate * duration_s) + t = np.linspace(0, duration_s, sample_count, endpoint=False) + wave = 0.6 * np.sin(2 * math.pi * frequency_hz * t) + pcm = np.clip(wave * np.iinfo(np.int16).max, -32768, 32767).astype(np.int16) + return base64.b64encode(pcm.tobytes()).decode("ascii") + + +def _wait_for(predicate: Callable[[], bool], timeout: float = 0.6) -> bool: + """Poll `predicate` until true or timeout.""" + end_time = time.time() + timeout + while time.time() < end_time: + if predicate(): + return True + time.sleep(0.01) + return False + + +def _start_wobbler() -> Tuple[HeadWobbler, List[Tuple[float, Tuple[float, float, float, float, float, float]]]]: + captured: List[Tuple[float, Tuple[float, float, float, float, float, float]]] = [] + + def capture(offsets: Tuple[float, float, float, float, float, float]) -> None: + captured.append((time.time(), offsets)) + + wobbler = HeadWobbler(set_speech_offsets=capture) + wobbler.start() + return wobbler, captured + + +def test_reset_drops_pending_offsets() -> None: + """Reset should stop wobble output derived from pre-reset audio.""" + wobbler, captured = _start_wobbler() + try: + wobbler.feed(_make_audio_chunk(duration_s=0.35)) + assert _wait_for(lambda: len(captured) > 0), "wobbler did not emit initial offsets" + + pre_reset_count = len(captured) + wobbler.reset() + time.sleep(0.3) + assert len(captured) == pre_reset_count, "offsets continued after reset without new audio" + finally: + wobbler.stop() + + +def test_reset_allows_future_offsets() -> None: + """After reset, fresh audio must still produce wobble offsets.""" + wobbler, captured = _start_wobbler() + try: + wobbler.feed(_make_audio_chunk(duration_s=0.35)) + assert _wait_for(lambda: len(captured) > 0), "wobbler did not emit initial offsets" + + wobbler.reset() + pre_second_count = len(captured) + + wobbler.feed(_make_audio_chunk(duration_s=0.35, frequency_hz=440.0)) + assert _wait_for(lambda: len(captured) > pre_second_count), "no offsets after reset" + assert wobbler._thread is not None and wobbler._thread.is_alive() + finally: + wobbler.stop() + + +def test_reset_during_inflight_chunk_keeps_worker(monkeypatch: Any) -> None: + """Simulate reset during chunk processing to ensure the worker survives.""" + wobbler, captured = _start_wobbler() + ready = threading.Event() + release = threading.Event() + + original_feed = wobbler.sway.feed + + def blocking_feed(pcm, sr): # type: ignore[no-untyped-def] + ready.set() + release.wait(timeout=2.0) + return original_feed(pcm, sr) + + monkeypatch.setattr(wobbler.sway, "feed", blocking_feed) + + try: + wobbler.feed(_make_audio_chunk(duration_s=0.35)) + assert ready.wait(timeout=1.0), "worker thread did not dequeue audio" + + wobbler.reset() + release.set() + + # Allow the worker to finish processing the first chunk (which should be discarded) + time.sleep(0.1) + + assert wobbler._thread is not None and wobbler._thread.is_alive(), "worker thread died after reset" + + pre_second = len(captured) + wobbler.feed(_make_audio_chunk(duration_s=0.35, frequency_hz=440.0)) + assert _wait_for(lambda: len(captured) > pre_second), "no offsets emitted after in-flight reset" + assert wobbler._thread.is_alive() + finally: + wobbler.stop() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..89f8fda03e380e48212ec51ca835d8f47177f885 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +"""Pytest configuration for path setup.""" + +import os +import sys +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SRC_PATH = PROJECT_ROOT / "src" +if str(SRC_PATH) not in sys.path: + sys.path.insert(0, str(SRC_PATH)) + + +# Make tests reproducible by ignoring machine-specific profile/tool env config. +# Without this, importing config during test collection can pick up a developer's +# local .env and fail before tests run. +os.environ["REACHY_MINI_SKIP_DOTENV"] = "1" +os.environ.pop("REACHY_MINI_CUSTOM_PROFILE", None) +os.environ.pop("REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY", None) +os.environ.pop("REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY", None) diff --git a/tests/test_config_name_collisions.py b/tests/test_config_name_collisions.py new file mode 100644 index 0000000000000000000000000000000000000000..0a2957da6051c373c3395401a5bd144c7e8bc531 --- /dev/null +++ b/tests/test_config_name_collisions.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import pytest + +import reachy_mini_receptionist.config as config_mod + + +def test_config_raises_on_external_profile_name_collision( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Config should fail fast when external/built-in profile names collide.""" + external_profiles = tmp_path / "external_profiles" + external_profiles.mkdir(parents=True) + (external_profiles / "default").mkdir() + + monkeypatch.setattr(config_mod.Config, "PROFILES_DIRECTORY", external_profiles) + monkeypatch.setattr(config_mod.Config, "TOOLS_DIRECTORY", None) + + with pytest.raises(RuntimeError, match="Ambiguous profile names"): + config_mod.Config() + + +def test_config_raises_on_external_tool_name_collision( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Config should fail fast when external/built-in tool names collide.""" + external_tools = tmp_path / "external_tools" + external_tools.mkdir(parents=True) + (external_tools / "dance.py").write_text("# collision with built-in dance tool\n", encoding="utf-8") + + monkeypatch.setattr(config_mod.Config, "PROFILES_DIRECTORY", config_mod.DEFAULT_PROFILES_DIRECTORY) + monkeypatch.setattr(config_mod.Config, "TOOLS_DIRECTORY", external_tools) + + with pytest.raises(RuntimeError, match="Ambiguous tool names"): + config_mod.Config() + + +def test_config_raises_when_selected_external_profile_is_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Config should fail fast when selected profile is absent from external root.""" + external_profiles = tmp_path / "external_profiles" + external_profiles.mkdir(parents=True) + + monkeypatch.setattr(config_mod.Config, "REACHY_MINI_CUSTOM_PROFILE", "missing_profile") + monkeypatch.setattr(config_mod.Config, "PROFILES_DIRECTORY", external_profiles) + monkeypatch.setattr(config_mod.Config, "TOOLS_DIRECTORY", None) + + with pytest.raises(RuntimeError, match="Selected profile 'missing_profile' was not found"): + config_mod.Config() diff --git a/tests/test_external_loading.py b/tests/test_external_loading.py new file mode 100644 index 0000000000000000000000000000000000000000..a8f048c84adffb876430116c5a6503ea5c292668 --- /dev/null +++ b/tests/test_external_loading.py @@ -0,0 +1,78 @@ +import sys +import importlib +from types import ModuleType +from pathlib import Path + +import pytest + +import reachy_mini_receptionist.config as config_mod + + +def _reload_core_tools() -> ModuleType: + """Reload core_tools after config object has been patched.""" + for module_name in list(sys.modules): + if module_name.startswith("reachy_mini_receptionist.tools."): + sys.modules.pop(module_name, None) + # External file-loaded modules are registered by bare tool name. + sys.modules.pop("ext_ping", None) + + sys.modules.pop("reachy_mini_receptionist.tools.core_tools", None) + core_tools_mod = importlib.import_module("reachy_mini_receptionist.tools.core_tools") + return core_tools_mod + + +def test_external_profile_can_use_builtin_tools( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """External profile tools.txt can reference built-in src tools.""" + profile_name = "ext_profile_test" + external_profiles_root = tmp_path / "external_profiles" + profile_dir = external_profiles_root / profile_name + profile_dir.mkdir(parents=True) + (profile_dir / "instructions.txt").write_text("hello\n", encoding="utf-8") + (profile_dir / "tools.txt").write_text("dance\n", encoding="utf-8") + + monkeypatch.setattr(config_mod.config, "REACHY_MINI_CUSTOM_PROFILE", profile_name) + monkeypatch.setattr(config_mod.config, "PROFILES_DIRECTORY", external_profiles_root) + monkeypatch.setattr(config_mod.config, "TOOLS_DIRECTORY", None) + monkeypatch.setattr(config_mod.config, "AUTOLOAD_EXTERNAL_TOOLS", False) + + core_tools_mod = _reload_core_tools() + + assert "dance" in core_tools_mod.ALL_TOOLS + + +def test_external_tools_can_be_loaded_without_external_profile( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """External tools can be loaded with built-in profile via autoload mode.""" + external_tools_root = tmp_path / "external_tools" + external_tools_root.mkdir(parents=True) + + (external_tools_root / "ext_ping.py").write_text( + "\n".join( + [ + "from typing import Any, Dict", + "from reachy_mini_receptionist.tools.core_tools import Tool, ToolDependencies", + "", + "class ExtPingTool(Tool):", + " name = \"ext_ping\"", + " description = \"External ping tool\"", + " parameters_schema = {\"type\": \"object\", \"properties\": {}, \"required\": []}", + "", + " async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]:", + " return {\"status\": \"ok\"}", + "", + ] + ), + encoding="utf-8", + ) + + monkeypatch.setattr(config_mod.config, "REACHY_MINI_CUSTOM_PROFILE", "default") + monkeypatch.setattr(config_mod.config, "PROFILES_DIRECTORY", config_mod.DEFAULT_PROFILES_DIRECTORY) + monkeypatch.setattr(config_mod.config, "TOOLS_DIRECTORY", external_tools_root) + monkeypatch.setattr(config_mod.config, "AUTOLOAD_EXTERNAL_TOOLS", True) + + core_tools_mod = _reload_core_tools() + + assert "ext_ping" in core_tools_mod.ALL_TOOLS diff --git a/tests/test_face_recognition_worker_state.py b/tests/test_face_recognition_worker_state.py new file mode 100644 index 0000000000000000000000000000000000000000..7ba6da0d86c32f22c3123963dceaed240aceb03e --- /dev/null +++ b/tests/test_face_recognition_worker_state.py @@ -0,0 +1,102 @@ +from typing import Any +from unittest.mock import MagicMock + +import pytest + +import reachy_mini_receptionist.face_recognition_worker as frw_mod +from reachy_mini_receptionist.face_recognition_worker import FaceRecognitionWorker + + +def _worker() -> FaceRecognitionWorker: + return FaceRecognitionWorker(face_db=MagicMock(), camera_worker=MagicMock()) + + +def test_known_face_requires_confirm_duration(monkeypatch: pytest.MonkeyPatch) -> None: + worker = _worker() + events: list[dict[str, Any]] = [] + worker.set_face_event_callback(events.append) + + now = 100.0 + + def _mono() -> float: + return now + + monkeypatch.setattr(frw_mod.time, "monotonic", _mono) + monkeypatch.setattr(frw_mod.time, "time", lambda: 1_700_000_000.0) + + worker._update_stable_state("known", "Alice", 48.0, 0.91) + assert worker._stable_state == "no_face" + assert events == [] + + now += 1.0 + worker._update_stable_state("known", "Alice", 47.0, 0.92) + assert worker._stable_state == "no_face" + assert events == [] + + now += 0.3 + worker._update_stable_state("known", "Alice", 46.0, 0.93) + assert worker._stable_state == "known" + assert worker.current_name == "Alice" + assert len(events) == 1 + assert events[0]["state"] == "known" + assert events[0]["name"] == "Alice" + + +def test_transition_within_cooldown_updates_state_but_skips_event(monkeypatch: pytest.MonkeyPatch) -> None: + worker = _worker() + events: list[dict[str, Any]] = [] + worker.set_face_event_callback(events.append) + + now = 200.0 + + def _mono() -> float: + return now + + monkeypatch.setattr(frw_mod.time, "monotonic", _mono) + monkeypatch.setattr(frw_mod.time, "time", lambda: 1_700_000_100.0) + + worker._update_stable_state("known", "Alice", 40.0, 0.95) + now += 1.3 + worker._update_stable_state("known", "Alice", 39.0, 0.95) + assert len(events) == 1 + assert worker._last_event_sent_at == pytest.approx(now) + + # New stable transition to unknown within 5s cooldown should not emit. + now += 1.3 + worker._update_stable_state("unknown", "Unknown", 130.0, 0.90) + now += 1.3 + worker._update_stable_state("unknown", "Unknown", 128.0, 0.88) + + assert worker._stable_state == "unknown" + assert worker.current_name == "Unknown" + assert len(events) == 1 + + +def test_no_face_uses_longer_grace_period(monkeypatch: pytest.MonkeyPatch) -> None: + worker = _worker() + + now = 300.0 + + def _mono() -> float: + return now + + monkeypatch.setattr(frw_mod.time, "monotonic", _mono) + monkeypatch.setattr(frw_mod.time, "time", lambda: 1_700_000_200.0) + + # First stabilize to known. + worker._update_stable_state("known", "Bob", 51.0, 0.94) + now += 1.3 + worker._update_stable_state("known", "Bob", 50.0, 0.94) + assert worker._stable_state == "known" + + # no_face should not stabilize before 2.5s confirmation. + now += 0.1 + worker._update_stable_state("no_face", "No face", 0.0, 0.0) + now += 2.4 + worker._update_stable_state("no_face", "No face", 0.0, 0.0) + assert worker._stable_state == "known" + + now += 0.2 + worker._update_stable_state("no_face", "No face", 0.0, 0.0) + assert worker._stable_state == "no_face" + assert worker.current_name == "Unknown" diff --git a/tests/test_openai_realtime.py b/tests/test_openai_realtime.py new file mode 100644 index 0000000000000000000000000000000000000000..7889da10fd6a6f298817794501de9c26a7aa6007 --- /dev/null +++ b/tests/test_openai_realtime.py @@ -0,0 +1,631 @@ +import random +import asyncio +import logging +from typing import Any +from datetime import datetime, timezone +from unittest.mock import MagicMock + +import pytest + +import reachy_mini_receptionist.openai_realtime as rt_mod +import reachy_mini_receptionist.tools.background_tool_manager as btm_mod +from reachy_mini_receptionist.openai_realtime import OpenaiRealtimeHandler, _compute_response_cost +from reachy_mini_receptionist.tools.core_tools import ToolDependencies +from reachy_mini_receptionist.tools.background_tool_manager import ToolCallRoutine + + +def _build_handler(loop: asyncio.AbstractEventLoop) -> OpenaiRealtimeHandler: + asyncio.set_event_loop(loop) + deps = ToolDependencies(reachy_mini=MagicMock(), movement_manager=MagicMock()) + return OpenaiRealtimeHandler(deps) + + +def test_format_timestamp_uses_wall_clock() -> None: + """Test that format_timestamp uses wall clock time.""" + loop = asyncio.new_event_loop() + try: + print("Testing format_timestamp...") + handler = _build_handler(loop) + formatted = handler.format_timestamp() + print(f"Formatted timestamp: {formatted}") + finally: + asyncio.set_event_loop(None) + loop.close() + + # Extract year from "[YYYY-MM-DD ...]" + year = int(formatted[1:5]) + assert year == datetime.now(timezone.utc).year + +@pytest.mark.asyncio +async def test_start_up_retries_on_abrupt_close(monkeypatch: Any, caplog: Any) -> None: + """First connection dies with ConnectionClosedError during iteration -> retried. + + Second connection iterates cleanly (no events) -> start_up returns without raising. + Ensures handler clears self.connection at the end. + """ + caplog.set_level(logging.WARNING) + + # Use a local Exception as the module's ConnectionClosedError to avoid ws dependency + FakeCCE = type("FakeCCE", (Exception,), {}) + monkeypatch.setattr(rt_mod, "ConnectionClosedError", FakeCCE) + + # Make asyncio.sleep return immediately (for backoff) + _real_sleep = asyncio.sleep + async def _mock_sleep(*_a: Any, **_kw: Any) -> None: await _real_sleep(0) + monkeypatch.setattr(asyncio, "sleep", _mock_sleep, raising=False) + + attempt_counter = {"n": 0} + + class FakeConn: + """Minimal realtime connection stub.""" + + def __init__(self, mode: str): + self._mode = mode + + class _Session: + async def update(self, **_kw: Any) -> None: return None + self.session = _Session() + + class _InputAudioBuffer: + async def append(self, **_kw: Any) -> None: return None + self.input_audio_buffer = _InputAudioBuffer() + + class _Item: + async def create(self, **_kw: Any) -> None: return None + + class _Conversation: + item = _Item() + self.conversation = _Conversation() + + class _Response: + async def create(self, **_kw: Any) -> None: return None + async def cancel(self, **_kw: Any) -> None: return None + self.response = _Response() + + async def __aenter__(self) -> "FakeConn": return self + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool: return False + async def close(self) -> None: return None + + # Async iterator protocol + def __aiter__(self) -> "FakeConn": return self + async def __anext__(self) -> None: + if self._mode == "raise_on_iter": + raise FakeCCE("abrupt close (simulated)") + raise StopAsyncIteration # clean exit (no events) + + class FakeRealtime: + def connect(self, **_kw: Any) -> FakeConn: + attempt_counter["n"] += 1 + mode = "raise_on_iter" if attempt_counter["n"] == 1 else "clean" + return FakeConn(mode) + + class FakeClient: + def __init__(self, **_kw: Any) -> None: self.realtime = FakeRealtime() + + # Patch the OpenAI client used by the handler + monkeypatch.setattr(rt_mod, "AsyncOpenAI", FakeClient) + + # Build handler with minimal deps + deps = ToolDependencies(reachy_mini=MagicMock(), movement_manager=MagicMock()) + handler = rt_mod.OpenaiRealtimeHandler(deps) + + # Run: should retry once and exit cleanly + await handler.start_up() + + # Validate: two attempts total (fail -> retry -> succeed), and connection cleared + assert attempt_counter["n"] == 2 + assert handler.connection is None + + # Optional: confirm we logged the unexpected close once + warnings = [r for r in caplog.records if r.levelname == "WARNING" and "closed unexpectedly" in r.msg] + assert len(warnings) == 1 + +# ---- Cost calculation tests ---- + + +def _make_usage( + audio_in: int | None = 0, + text_in: int | None = 0, + image_in: int | None = 0, + audio_out: int | None = 0, + text_out: int | None = 0, + has_input: bool = True, + has_output: bool = True, +) -> MagicMock: + """Build a fake usage object matching the OpenAI response.usage shape.""" + usage = MagicMock() + if has_input: + inp = MagicMock() + inp.audio_tokens = audio_in + inp.text_tokens = text_in + inp.image_tokens = image_in + usage.input_token_details = inp + else: + usage.input_token_details = None + if has_output: + out = MagicMock() + out.audio_tokens = audio_out + out.text_tokens = text_out + usage.output_token_details = out + else: + usage.output_token_details = None + return usage + + +@pytest.mark.parametrize( + "usage_kwargs, expect_positive", + [ + # All token types present → positive cost + ({"audio_in": 1000, "text_in": 2000, "image_in": 500, "audio_out": 800, "text_out": 300}, True), + # All None tokens → must not crash + ({"audio_in": None, "text_in": None, "image_in": None, "audio_out": None, "text_out": None}, False), + # Mix of None and valid ints + ({"audio_in": None, "text_in": 500, "image_in": None, "audio_out": 1000, "text_out": None}, True), + # Missing input/output details entirely + ({"has_input": False, "has_output": False}, False), + ], + ids=["normal", "all_none", "mixed", "missing_details"], +) +def test_compute_response_cost(usage_kwargs: dict[str, Any], expect_positive: bool) -> None: + """Verify _compute_response_cost handles various token combinations without crashing.""" + usage = _make_usage(**usage_kwargs) + cost = _compute_response_cost(usage) + if expect_positive: + assert cost > 0 + else: + assert cost == 0.0 + + +# ---- Stress test: response.create rejection + retry ---- + + +@pytest.mark.asyncio +async def test_response_sender_retries_on_active_response_rejection(monkeypatch: Any, caplog: Any) -> None: + """Stress test: response.create rejection + retry via real event processing. + + Tool results (is_idle_tool_call=False) queue response.create calls via + _safe_response_create. When the server rejects some with + ``conversation_already_has_active_response``, the error event flows through + the event handler and _response_sender_loop retries the rejected request. + + The full _run_realtime_session event loop runs so that the error-handling + code path (setting _last_response_rejected) is exercised by real event + processing, not mocked out. + """ + caplog.set_level(logging.DEBUG) + + FakeCCE = type("FakeCCE", (Exception,), {}) + monkeypatch.setattr(rt_mod, "ConnectionClosedError", FakeCCE) + monkeypatch.setattr(rt_mod, "get_session_instructions", lambda: "test") + monkeypatch.setattr(rt_mod, "get_session_voice", lambda: "alloy") + monkeypatch.setattr(rt_mod, "get_tool_specs", lambda: []) + + N_TOOL_RESULTS = 400 + REJECT_CALL_NUMBERS = {1, 3, 5, 10, 25, 50, 75, 100, 150, 200, 300, 399} + EXPECTED_TOTAL_CALLS = N_TOOL_RESULTS + len(REJECT_CALL_NUMBERS) + + event_queue: asyncio.Queue[Any] = asyncio.Queue() + response_create_log: list[tuple[int, dict[str, Any]]] = [] + handler_ref: list[Any] = [] + + # ---- Fake event / error objects mirroring the OpenAI SDK shapes ---- + + class FakeError: + def __init__(self, message: str, code: str) -> None: + self.message = message + self.code = code + self.type = "invalid_request_error" + self.event_id = None + self.param = None + + def __repr__(self) -> str: + return ( + f"RealtimeError(message='{self.message}', type='{self.type}', " + f"code='{self.code}', event_id=None, param=None)" + ) + + class FakeEvent: + def __init__(self, etype: str, **kwargs: Any) -> None: + self.type = etype + for k, v in kwargs.items(): + setattr(self, k, v) + + # ---- Fake connection components ---- + + class FakeResponseAPI: + """Mimics connection.response. + + Pushes server events into the shared event_queue so they flow + through the real event-handling code. Also guards the serialization + invariant: every create() must arrive when no response is active. + """ + + def __init__(self) -> None: + self._call_count = 0 + self._serialization_violations: list[int] = [] + + async def create(self, **kwargs: Any) -> None: + self._call_count += 1 + n = self._call_count + response_create_log.append((n, kwargs)) + + h = handler_ref[0] + + # Real backend rejects when a response is already active. + if not h._response_done_event.is_set(): + self._serialization_violations.append(n) + await event_queue.put( + FakeEvent( + "error", + error=FakeError( + message=( + f"Conversation already has an active response in " + f"progress: resp_fake{n}. Wait until the response " + f"is finished before creating a new one." + ), + code="conversation_already_has_active_response", + ), + ) + ) + await asyncio.sleep(0) + await event_queue.put( + FakeEvent("response.done", response=MagicMock()) + ) + return + + # Intentional rejections (simulating a race where another + # response sneaks in right after our check). + if n in REJECT_CALL_NUMBERS: + await event_queue.put( + FakeEvent( + "error", + error=FakeError( + message=( + f"Conversation already has an active response in " + f"progress: resp_fake{n}. Wait until the response " + f"is finished before creating a new one." + ), + code="conversation_already_has_active_response", + ), + ) + ) + await asyncio.sleep(0) + else: + await event_queue.put(FakeEvent("response.created")) + + await event_queue.put( + FakeEvent("response.done", response=MagicMock()) + ) + + + async def cancel(self, **_kw: Any) -> None: + pass + + fake_response_api = FakeResponseAPI() + + class FakeSession: + async def update(self, **_kw: Any) -> None: + pass + + class FakeInputAudioBuffer: + async def append(self, **_kw: Any) -> None: + pass + + class FakeItem: + async def create(self, **_kw: Any) -> None: + pass + + class FakeConversation: + item = FakeItem() + + class FakeConn: + session = FakeSession() + input_audio_buffer = FakeInputAudioBuffer() + conversation = FakeConversation() + response = fake_response_api + + async def __aenter__(self) -> "FakeConn": + return self + + async def __aexit__(self, *_a: Any) -> bool: + return False + + async def close(self) -> None: + pass + + def __aiter__(self) -> "FakeConn": + return self + + async def __anext__(self) -> FakeEvent: + event: FakeEvent = await event_queue.get() + if event is None: # sentinel → end iteration + raise StopAsyncIteration + return event + + class FakeRealtime: + def connect(self, **_kw: Any) -> FakeConn: + return FakeConn() + + class FakeClient: + def __init__(self, **_kw: Any) -> None: + self.realtime = FakeRealtime() + + monkeypatch.setattr(rt_mod, "AsyncOpenAI", FakeClient) + + # Patch dispatch_tool_call so tools complete with a result. + async def _fake_dispatch( + tool_name: str, args_json: str, deps: Any, **_kw: Any + ) -> dict[str, Any]: + await asyncio.sleep(random.uniform(0.3, 0.5)) + return {"ok": True, "tool": tool_name} + + monkeypatch.setattr(btm_mod, "dispatch_tool_call", _fake_dispatch) + + # ---- Build handler and start the full realtime session ---- + + deps = ToolDependencies(reachy_mini=MagicMock(), movement_manager=MagicMock()) + handler = rt_mod.OpenaiRealtimeHandler(deps) + handler_ref.append(handler) + + asyncio.create_task(handler.start_up()) + + # ---- Start tools via the real BackgroundToolManager pipeline ---- + # start_tool → _run_tool → notification queue → listener → _handle_tool_result + + for i in range(N_TOOL_RESULTS): + await handler.tool_manager.start_tool( + call_id=f"call_{i}", + tool_call_routine=ToolCallRoutine( + tool_name="test_tool", + args_json_str=f'{{"index": {i}}}', + deps=deps, + ), + is_idle_tool_call=False, + ) + + # Yield so spawned tool tasks, the listener, and the sender can drain. + await asyncio.sleep(5) + + # ---- Tear down ---- + + await event_queue.put(None) # sentinel stops event iteration + + await handler.shutdown() + + + # ---- Assertions ---- + + # Serialization: every response.create() must have been called only when + # no response was in-flight (_response_done_event was set). Any violation + # means the sender fired a new request before the previous one finished. + assert fake_response_api._serialization_violations == [], ( + f"response.create() was called while a response was still active on " + f"call(s) {fake_response_api._serialization_violations}" + ) + + # Total response.create() calls = tool results + retries for rejected ones + assert fake_response_api._call_count == EXPECTED_TOTAL_CALLS, ( + f"Expected {EXPECTED_TOTAL_CALLS} response.create calls " + f"({N_TOOL_RESULTS} results + {len(REJECT_CALL_NUMBERS)} retries), " + f"got {fake_response_api._call_count}" + ) + + # The error event handler must have set _last_response_rejected for each + # rejection (the log message comes from the event handler code path). + rejection_logs = [ + r for r in caplog.records + if "worker will retry" in getattr(r, "msg", "") + ] + assert len(rejection_logs) == len(REJECT_CALL_NUMBERS), ( + f"Expected {len(REJECT_CALL_NUMBERS)} rejection entries from error handler, " + f"got {len(rejection_logs)}" + ) + + # The sender loop must have retried after each rejection. + retry_logs = [ + r for r in caplog.records + if "response.create was rejected; retrying" in getattr(r, "msg", "") + ] + assert len(retry_logs) == len(REJECT_CALL_NUMBERS), ( + f"Expected {len(REJECT_CALL_NUMBERS)} retry entries from sender loop, " + f"got {len(retry_logs)}" + ) + + +# ---- Response creation timeout guard tests ---- + + +@pytest.mark.asyncio +async def test_response_sender_loop_times_out_waiting_for_response_done( + monkeypatch: Any, caplog: Any, +) -> None: + """If response.done is never received the sender loop should time out. + + Rather than hang forever, it force-sets the event and moves on. + """ + caplog.set_level(logging.DEBUG) + + monkeypatch.setattr(rt_mod, "_RESPONSE_DONE_TIMEOUT", 0.3) + + deps = ToolDependencies(reachy_mini=MagicMock(), movement_manager=MagicMock()) + handler = rt_mod.OpenaiRealtimeHandler(deps) + + create_count = 0 + + class FakeResponse: + async def create(self, **_kw: Any) -> None: + nonlocal create_count + create_count += 1 + # Simulate response.created clearing the event, but never + # send response.done (so the event stays cleared forever). + handler._response_done_event.clear() + + async def cancel(self, **_kw: Any) -> None: + pass + + fake_conn = MagicMock() + fake_conn.response = FakeResponse() + handler.connection = fake_conn + + # Queue two requests + await handler._safe_response_create(instructions="req1") + await handler._safe_response_create(instructions="req2") + + sender_task = asyncio.create_task(handler._response_sender_loop()) + + # Give enough time for both requests to time out (0.3s each + margin) + await asyncio.sleep(1.5) + + handler.connection = None # signal the loop to exit + handler._response_done_event.set() + await asyncio.wait_for(sender_task, timeout=2.0) + + assert create_count == 2, f"Expected 2 response.create calls, got {create_count}" + + timeout_logs = [ + r for r in caplog.records + if "Timed out waiting for response.done" in r.getMessage() + ] + assert len(timeout_logs) == 2, ( + f"Expected 2 timeout warnings, got {len(timeout_logs)}" + ) + + +@pytest.mark.asyncio +async def test_response_sender_loop_times_out_waiting_for_previous_response( + monkeypatch: Any, caplog: Any, +) -> None: + """If a previous response never completes, the pre-condition wait times out. + + It should force-set the event and proceed to send. + """ + caplog.set_level(logging.DEBUG) + + monkeypatch.setattr(rt_mod, "_RESPONSE_DONE_TIMEOUT", 0.3) + + deps = ToolDependencies(reachy_mini=MagicMock(), movement_manager=MagicMock()) + handler = rt_mod.OpenaiRealtimeHandler(deps) + + # Pretend a response is already in-flight (event cleared) + handler._response_done_event.clear() + + created = asyncio.Event() + + class FakeResponse: + async def create(self, **_kw: Any) -> None: + # Immediately complete the response cycle so the loop can finish + handler._response_done_event.set() + created.set() + + async def cancel(self, **_kw: Any) -> None: + pass + + fake_conn = MagicMock() + fake_conn.response = FakeResponse() + handler.connection = fake_conn + + await handler._safe_response_create(instructions="waiting_req") + + sender_task = asyncio.create_task(handler._response_sender_loop()) + + # Wait for the request to be sent (after timing out on the pre-condition) + await asyncio.wait_for(created.wait(), timeout=2.0) + + handler.connection = None + handler._response_done_event.set() + await asyncio.wait_for(sender_task, timeout=2.0) + + timeout_logs = [ + r for r in caplog.records + if "Timed out waiting for previous response" in r.getMessage() + ] + assert len(timeout_logs) == 1, ( + f"Expected 1 pre-condition timeout warning, got {len(timeout_logs)}" + ) + + +@pytest.mark.asyncio +async def test_push_face_context_event_creates_conversation_item_only() -> None: + """Face context updates should be pushed as conversation context only. + + This path must not implicitly create a model response. + """ + deps = ToolDependencies(reachy_mini=MagicMock(), movement_manager=MagicMock()) + handler = rt_mod.OpenaiRealtimeHandler(deps) + + created_items: list[dict[str, Any]] = [] + + class FakeItem: + async def create(self, **kwargs: Any) -> None: + created_items.append(kwargs) + + class FakeConversation: + item = FakeItem() + + class FakeResponse: + async def create(self, **_kwargs: Any) -> None: + raise AssertionError("response.create must not be called for face context events") + + fake_conn = MagicMock() + fake_conn.conversation = FakeConversation() + fake_conn.response = FakeResponse() + handler.connection = fake_conn + + await handler._push_face_context_event( + { + "state": "known", + "name": "Beyonce", + "previous_state": "unknown", + "previous_name": None, + "lbph_confidence": 41.2, + "detection_confidence": 0.94, + } + ) + + assert len(created_items) == 1 + item = created_items[0]["item"] + assert item["type"] == "message" + assert item["role"] == "user" + text = item["content"][0]["text"] + assert "state=known" in text + assert "name=Beyonce" in text + + +@pytest.mark.asyncio +async def test_notify_external_face_event_schedules_context_push() -> None: + """Thread-safe notifier should schedule and deliver a conversation item push.""" + deps = ToolDependencies(reachy_mini=MagicMock(), movement_manager=MagicMock()) + handler = rt_mod.OpenaiRealtimeHandler(deps) + handler._runtime_loop = asyncio.get_running_loop() + + created_items: list[dict[str, Any]] = [] + + class FakeItem: + async def create(self, **kwargs: Any) -> None: + created_items.append(kwargs) + + class FakeConversation: + item = FakeItem() + + fake_conn = MagicMock() + fake_conn.conversation = FakeConversation() + handler.connection = fake_conn + + handler.notify_external_face_event( + { + "state": "unknown", + "name": None, + "previous_state": "no_face", + "previous_name": None, + "lbph_confidence": 0.0, + "detection_confidence": 0.88, + } + ) + + await asyncio.sleep(0.05) + + assert len(created_items) == 1 + item = created_items[0]["item"] + assert item["type"] == "message" + assert item["role"] == "user" + assert "state=unknown" in item["content"][0]["text"] diff --git a/tests/tools/test_background_tool_manager.py b/tests/tools/test_background_tool_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b55d297b81885b127ac41e8dd197d7a0f54cfbec --- /dev/null +++ b/tests/tools/test_background_tool_manager.py @@ -0,0 +1,545 @@ +"""Tests for BackgroundToolManager.""" + +from __future__ import annotations +import asyncio +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from reachy_mini_receptionist.tools.tool_constants import ToolState +from reachy_mini_receptionist.tools.background_tool_manager import ( + ToolProgress, + BackgroundTool, + ToolCallRoutine, + ToolNotification, + BackgroundToolManager, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_routine( + tool_name: str = "test_tool", + result: dict[str, Any] | None = None, + error: Exception | None = None, + delay: float = 0.0, +) -> ToolCallRoutine: + """Create a mock ToolCallRoutine that returns *result* or raises *error*. + + If *delay* > 0, the routine will sleep for that many seconds before + returning / raising so we can test cancellation and progress. + + Mirrors the contract of ``_dispatch_tool_call`` in core_tools: exceptions + (including ``CancelledError``) are caught and returned as + ``{"error": "..."}`` dicts so that ``_run_tool`` never sees a raw raise. + """ + routine = MagicMock(spec=ToolCallRoutine) + routine.tool_name = tool_name + routine.args_json_str = "{}" + + async def _call(manager: BackgroundToolManager) -> dict[str, Any]: + try: + if delay: + await asyncio.sleep(delay) + if error is not None: + raise error + return result or {"ok": True} + except asyncio.CancelledError: + return {"error": "Tool cancelled"} + except Exception as e: + return {"error": f"{type(e).__name__}: {e}"} + + routine.__call__ = _call # type: ignore[method-assign] + routine.side_effect = _call + return routine + + +# --------------------------------------------------------------------------- +# Model / data-class sanity checks +# --------------------------------------------------------------------------- + + +class TestToolProgress: + """Validate ToolProgress construction and bounds.""" + + def test_valid_progress(self) -> None: + """Accept valid progress values and messages.""" + p = ToolProgress(progress=0.5, message="halfway") + assert p.progress == 0.5 + assert p.message == "halfway" + + def test_bounds(self) -> None: + """Allow 0.0 and 1.0 as boundary values.""" + assert ToolProgress(progress=0.0).progress == 0.0 + assert ToolProgress(progress=1.0).progress == 1.0 + + def test_out_of_bounds_raises(self) -> None: + """Reject progress values outside [0, 1].""" + with pytest.raises(Exception): + ToolProgress(progress=-0.1) + with pytest.raises(Exception): + ToolProgress(progress=1.1) + + +class TestToolNotification: + """Validate ToolNotification construction.""" + + def test_creation(self) -> None: + """Create a notification and verify its fields.""" + n = ToolNotification( + id="abc", + tool_name="my_tool", + is_idle_tool_call=False, + status=ToolState.COMPLETED, + result={"data": 1}, + ) + assert n.id == "abc" + assert n.status == ToolState.COMPLETED + assert n.result == {"data": 1} + assert n.error is None + + +class TestBackgroundTool: + """Validate BackgroundTool helpers.""" + + def test_tool_id(self) -> None: + """Verify the composite tool_id property includes started_at.""" + t = BackgroundTool( + id="123", + tool_name="weather", + is_idle_tool_call=False, + status=ToolState.RUNNING, + ) + assert t.tool_id == f"weather-123-{t.started_at}" + + def test_get_notification(self) -> None: + """Convert a BackgroundTool to a ToolNotification.""" + t = BackgroundTool( + id="1", + tool_name="t", + is_idle_tool_call=True, + status=ToolState.COMPLETED, + result={"x": 1}, + error=None, + ) + n = t.get_notification() + assert isinstance(n, ToolNotification) + assert n.id == "1" + assert n.tool_name == "t" + assert n.is_idle_tool_call is True + assert n.status == ToolState.COMPLETED + assert n.result == {"x": 1} + + +# --------------------------------------------------------------------------- +# BackgroundToolManager +# --------------------------------------------------------------------------- + + +@pytest.fixture +def manager() -> BackgroundToolManager: + """Return a fresh BackgroundToolManager for each test.""" + return BackgroundToolManager() + + +class TestSetLoop: + """Verify event-loop assignment via set_loop.""" + + @pytest.mark.asyncio + async def test_set_loop_uses_running_loop(self, manager: BackgroundToolManager) -> None: + """Default to the current running loop.""" + manager.set_loop() + assert manager._loop is asyncio.get_running_loop() + + def test_set_loop_explicit(self, manager: BackgroundToolManager) -> None: + """Accept an explicitly provided loop.""" + loop = asyncio.new_event_loop() + try: + manager.set_loop(loop) + assert manager._loop is loop + finally: + loop.close() + + def test_set_loop_creates_new_when_no_running(self, manager: BackgroundToolManager) -> None: + """When called outside an async context it falls back to a new loop.""" + manager.set_loop() + assert manager._loop is not None + + +class TestStartTool: + """Verify tool registration via start_tool.""" + + @pytest.mark.asyncio + async def test_start_registers_tool(self, manager: BackgroundToolManager) -> None: + """Register a tool and verify its initial state.""" + routine = _make_routine("greet") + bg = await manager.start_tool( + call_id="c1", + tool_call_routine=routine, + is_idle_tool_call=False, + ) + assert bg.tool_name == "greet" + assert bg.id == "c1" + assert bg.status == ToolState.RUNNING + assert manager.get_tool(bg.tool_id) is bg + + # Let the task finish + await asyncio.sleep(0.05) + + @pytest.mark.asyncio + async def test_start_with_progress(self, manager: BackgroundToolManager) -> None: + """Initialize progress tracking when requested.""" + routine = _make_routine("slow", delay=0.1) + bg = await manager.start_tool( + call_id="c2", + tool_call_routine=routine, + is_idle_tool_call=True, + with_progress=True, + ) + assert bg.progress is not None + assert bg.progress.progress == 0.0 + await asyncio.sleep(0.15) + + +class TestRunToolLifecycle: + """Test _run_tool via start_tool (the public entry point).""" + + @pytest.mark.asyncio + async def test_successful_completion(self, manager: BackgroundToolManager) -> None: + """Complete a tool and verify result, status, and notification.""" + routine = _make_routine("ok_tool", result={"answer": 42}) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + + # Wait for the task to finish + await asyncio.sleep(0.05) + + assert bg.status == ToolState.COMPLETED + assert bg.result == {"answer": 42} + assert bg.completed_at is not None + assert bg.error is None + + # Notification should be queued + notification = manager._notification_queue.get_nowait() + assert notification.status == ToolState.COMPLETED + + @pytest.mark.asyncio + async def test_tool_failure(self, manager: BackgroundToolManager) -> None: + """Mark a tool as FAILED when it raises an exception.""" + routine = _make_routine("bad_tool", error=ValueError("boom")) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + + await asyncio.sleep(0.05) + + assert bg.status == ToolState.FAILED + assert "ValueError: boom" in (bg.error or "") + assert bg.completed_at is not None + + notification = manager._notification_queue.get_nowait() + assert notification.status == ToolState.FAILED + + @pytest.mark.asyncio + async def test_tool_cancellation(self, manager: BackgroundToolManager) -> None: + """Cancel a running tool and verify CANCELLED status.""" + routine = _make_routine("long_tool", delay=10.0) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + + # Give the task a moment to start, then cancel + await asyncio.sleep(0.02) + cancelled = await manager.cancel_tool(bg.tool_id) + assert cancelled is True + + # Let cancellation propagate + await asyncio.sleep(0.05) + + assert bg.status == ToolState.CANCELLED + assert bg.error == "Tool cancelled" + assert bg.completed_at is not None + + +class TestUpdateProgress: + """Verify progress updates on running tools.""" + + @pytest.mark.asyncio + async def test_update_progress_success(self, manager: BackgroundToolManager) -> None: + """Update progress value and message on a tracked tool.""" + routine = _make_routine("prog", delay=0.5) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False, with_progress=True) + + ok = await manager.update_progress(bg.tool_id, 0.5, "half done") + assert ok is True + assert bg.progress is not None + assert bg.progress.progress == 0.5 + assert bg.progress.message == "half done" + + # Cancel to clean up + await manager.cancel_tool(bg.tool_id) + await asyncio.sleep(0.05) + + @pytest.mark.asyncio + async def test_update_progress_clamps(self, manager: BackgroundToolManager) -> None: + """Clamp out-of-range progress values to [0, 1].""" + routine = _make_routine("prog", delay=0.5) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False, with_progress=True) + + await manager.update_progress(bg.tool_id, 1.5) + assert bg.progress is not None + assert bg.progress.progress == 1.0 + + await manager.update_progress(bg.tool_id, -0.5) + assert bg.progress.progress == 0.0 + + await manager.cancel_tool(bg.tool_id) + await asyncio.sleep(0.05) + + @pytest.mark.asyncio + async def test_update_progress_unknown_tool(self, manager: BackgroundToolManager) -> None: + """Return False for an unknown tool_id.""" + ok = await manager.update_progress("nonexistent", 0.5) + assert ok is False + + @pytest.mark.asyncio + async def test_update_progress_no_tracking(self, manager: BackgroundToolManager) -> None: + """Return False when progress tracking is disabled.""" + routine = _make_routine("fast", delay=0.5) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False, with_progress=False) + + ok = await manager.update_progress(bg.tool_id, 0.5) + assert ok is False + + await manager.cancel_tool(bg.tool_id) + await asyncio.sleep(0.05) + + +class TestCancelTool: + """Verify tool cancellation behaviour.""" + + @pytest.mark.asyncio + async def test_cancel_nonexistent(self, manager: BackgroundToolManager) -> None: + """Return False when the tool_id does not exist.""" + result = await manager.cancel_tool("does-not-exist") + assert result is False + + @pytest.mark.asyncio + async def test_cancel_already_completed(self, manager: BackgroundToolManager) -> None: + """Return True when cancelling an already-completed tool.""" + routine = _make_routine("done") + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + await asyncio.sleep(0.05) # let it finish + assert bg.status == ToolState.COMPLETED + + # Cancelling a completed tool should return True (not running, no-op) + result = await manager.cancel_tool(bg.tool_id) + assert result is True + + +class TestTimeoutTools: + """Verify automatic timeout of long-running tools.""" + + @pytest.mark.asyncio + async def test_timeout_cancels_old_tools(self, manager: BackgroundToolManager) -> None: + """Cancel tools exceeding max duration.""" + # Use a very short max duration + manager._max_tool_duration_seconds = 0.01 + + routine = _make_routine("slow", delay=10.0) + await manager.start_tool("c1", routine, is_idle_tool_call=False) + + # Wait longer than the timeout + await asyncio.sleep(0.05) + + count = await manager.timeout_tools() + assert count == 1 + + await asyncio.sleep(0.05) + + @pytest.mark.asyncio + async def test_timeout_ignores_recent_tools(self, manager: BackgroundToolManager) -> None: + """Leave recent tools untouched.""" + manager._max_tool_duration_seconds = 9999 + + routine = _make_routine("fast", delay=10.0) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + + count = await manager.timeout_tools() + assert count == 0 + + await manager.cancel_tool(bg.tool_id) + await asyncio.sleep(0.05) + + +class TestCleanupTools: + """Verify cleanup of completed tools from memory.""" + + @pytest.mark.asyncio + async def test_cleanup_removes_old_completed(self, manager: BackgroundToolManager) -> None: + """Remove completed tools past the retention window.""" + manager._max_tool_memory_seconds = 0.01 + + routine = _make_routine("old") + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + await asyncio.sleep(0.05) + assert bg.status == ToolState.COMPLETED + + # Wait for the memory retention to expire + await asyncio.sleep(0.05) + + removed = await manager.cleanup_tools() + assert removed == 1 + assert manager.get_tool(bg.tool_id) is None + + @pytest.mark.asyncio + async def test_cleanup_keeps_recent_completed(self, manager: BackgroundToolManager) -> None: + """Keep recently completed tools.""" + manager._max_tool_memory_seconds = 9999 + + routine = _make_routine("recent") + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + await asyncio.sleep(0.05) + + removed = await manager.cleanup_tools() + assert removed == 0 + assert manager.get_tool(bg.tool_id) is not None + + @pytest.mark.asyncio + async def test_cleanup_ignores_running(self, manager: BackgroundToolManager) -> None: + """Never remove still-running tools.""" + manager._max_tool_memory_seconds = 0.0 # immediate expiry + + routine = _make_routine("still_going", delay=10.0) + bg = await manager.start_tool("c1", routine, is_idle_tool_call=False) + + removed = await manager.cleanup_tools() + assert removed == 0 + + await manager.cancel_tool(bg.tool_id) + await asyncio.sleep(0.05) + + +class TestGetters: + """Verify tool retrieval helpers.""" + + @pytest.mark.asyncio + async def test_get_tool(self, manager: BackgroundToolManager) -> None: + """Return None for missing tools and the instance for known ones.""" + assert manager.get_tool("nope") is None + + routine = _make_routine("x") + bg = await manager.start_tool("1", routine, is_idle_tool_call=False) + assert manager.get_tool(bg.tool_id) is bg + await asyncio.sleep(0.05) + + @pytest.mark.asyncio + async def test_get_running_tools(self, manager: BackgroundToolManager) -> None: + """Return only tools that are still running.""" + r1 = _make_routine("a", delay=10.0) + r2 = _make_routine("b", delay=10.0) + r3 = _make_routine("c") # finishes immediately + + bg1 = await manager.start_tool("1", r1, is_idle_tool_call=False) + bg2 = await manager.start_tool("2", r2, is_idle_tool_call=False) + await manager.start_tool("3", r3, is_idle_tool_call=False) + await asyncio.sleep(0.05) # let r3 finish + + running = manager.get_running_tools() + assert len(running) == 2 + names = {t.tool_name for t in running} + assert names == {"a", "b"} + + # Clean up + await manager.cancel_tool(bg1.tool_id) + await manager.cancel_tool(bg2.tool_id) + await asyncio.sleep(0.05) + + @pytest.mark.asyncio + async def test_get_all_tools_sorted(self, manager: BackgroundToolManager) -> None: + """Tools are returned most-recent-first.""" + r1 = _make_routine("first") + r2 = _make_routine("second") + + await manager.start_tool("1", r1, is_idle_tool_call=False) + await asyncio.sleep(0.02) # ensure different started_at + await manager.start_tool("2", r2, is_idle_tool_call=False) + + await asyncio.sleep(0.05) + + all_tools = manager.get_all_tools() + assert len(all_tools) == 2 + assert all_tools[0].tool_name == "second" + assert all_tools[1].tool_name == "first" + + @pytest.mark.asyncio + async def test_get_all_tools_limit(self, manager: BackgroundToolManager) -> None: + """Respect the limit parameter on get_all_tools.""" + for i in range(5): + r = _make_routine(f"t{i}") + await manager.start_tool(str(i), r, is_idle_tool_call=False) + + await asyncio.sleep(0.05) + + limited = manager.get_all_tools(limit=3) + assert len(limited) == 3 + + +class TestStartUp: + """Verify start_up bootstraps background tasks.""" + + @pytest.mark.asyncio + async def test_startup_creates_tasks(self, manager: BackgroundToolManager) -> None: + """start_up should create the listener and cleanup background tasks.""" + callback = AsyncMock() + manager.start_up(tool_callbacks=[callback]) + + # Start a tool and let it complete — the listener should invoke the callback + routine = _make_routine("ping") + await manager.start_tool("c1", routine, is_idle_tool_call=False) + await asyncio.sleep(0.1) + + assert callback.call_count == 1 + notification = callback.call_args[0][0] + assert isinstance(notification, ToolNotification) + assert notification.status == ToolState.COMPLETED + + @pytest.mark.asyncio + async def test_startup_multiple_callbacks(self, manager: BackgroundToolManager) -> None: + """Invoke all registered callbacks on completion.""" + cb1 = AsyncMock() + cb2 = AsyncMock() + manager.start_up(tool_callbacks=[cb1, cb2]) + + routine = _make_routine("multi") + await manager.start_tool("c1", routine, is_idle_tool_call=False) + await asyncio.sleep(0.1) + + assert cb1.call_count == 1 + assert cb2.call_count == 1 + + +class TestNotificationQueue: + """Verify notifications are enqueued on tool completion or failure.""" + + @pytest.mark.asyncio + async def test_notifications_queued_on_completion(self, manager: BackgroundToolManager) -> None: + """Queue a COMPLETED notification with the tool result.""" + routine = _make_routine("notif", result={"v": 1}) + await manager.start_tool("c1", routine, is_idle_tool_call=False) + await asyncio.sleep(0.05) + + n = manager._notification_queue.get_nowait() + assert n.tool_name == "notif" + assert n.status == ToolState.COMPLETED + assert n.result == {"v": 1} + + @pytest.mark.asyncio + async def test_notifications_queued_on_failure(self, manager: BackgroundToolManager) -> None: + """Queue a FAILED notification with the error message.""" + routine = _make_routine("fail", error=RuntimeError("oops")) + await manager.start_tool("c1", routine, is_idle_tool_call=False) + await asyncio.sleep(0.05) + + n = manager._notification_queue.get_nowait() + assert n.status == ToolState.FAILED + assert "RuntimeError: oops" in (n.error or "") diff --git a/tests/vision/test_processors.py b/tests/vision/test_processors.py new file mode 100644 index 0000000000000000000000000000000000000000..7728802de9aec2bc10226db38ee760229bf095b6 --- /dev/null +++ b/tests/vision/test_processors.py @@ -0,0 +1,498 @@ +"""Tests for the vision processing module.""" + +import time +from typing import Any +from unittest.mock import Mock, MagicMock, patch + +import numpy as np +import pytest + +from reachy_mini_receptionist.vision.processors import ( + VisionConfig, + VisionManager, + VisionProcessor, + initialize_vision_manager, +) + + +def test_vision_config_defaults() -> None: + """Test VisionConfig has sensible defaults.""" + config = VisionConfig() + assert config.vision_interval == 5.0 + assert config.max_new_tokens == 64 + assert config.jpeg_quality == 85 + assert config.max_retries == 3 + assert config.retry_delay == 1.0 + assert config.device_preference == "auto" + + +def test_vision_config_custom_values() -> None: + """Test VisionConfig accepts custom values.""" + config = VisionConfig( + model_path="/custom/path", + vision_interval=10.0, + max_new_tokens=128, + jpeg_quality=95, + max_retries=5, + retry_delay=2.0, + device_preference="cpu", + ) + assert config.model_path == "/custom/path" + assert config.vision_interval == 10.0 + assert config.max_new_tokens == 128 + assert config.jpeg_quality == 95 + assert config.max_retries == 5 + assert config.retry_delay == 2.0 + assert config.device_preference == "cpu" + + + +@pytest.fixture +def mock_torch() -> Any: + """Mock torch module to avoid loading actual models.""" + with patch("reachy_mini_receptionist.vision.processors.torch") as mock: + mock.cuda.is_available.return_value = False + mock.backends.mps.is_available.return_value = False + mock.float32 = "float32" + mock.bfloat16 = "bfloat16" + yield mock + + +@pytest.fixture +def mock_transformers() -> Any: + """Mock transformers module.""" + with patch("reachy_mini_receptionist.vision.processors.AutoProcessor") as proc, \ + patch("reachy_mini_receptionist.vision.processors.AutoModelForImageTextToText") as model: + + # Mock processor + mock_processor = MagicMock() + mock_processor.apply_chat_template.return_value = { + "input_ids": MagicMock(to=lambda x: MagicMock()), + "attention_mask": MagicMock(to=lambda x: MagicMock()), + "pixel_values": MagicMock(to=lambda x: MagicMock()), + } + mock_processor.batch_decode.return_value = ["assistant\nThis is a test description."] + mock_processor.tokenizer.eos_token_id = 2 + proc.from_pretrained.return_value = mock_processor + + # Mock model + mock_model_instance = MagicMock() + mock_model_instance.eval.return_value = None + mock_model_instance.generate.return_value = [[1, 2, 3]] + mock_model_instance.to.return_value = mock_model_instance + model.from_pretrained.return_value = mock_model_instance + + yield {"processor": proc, "model": model} + + +def test_vision_processor_device_selection_cpu(mock_torch: Any) -> None: + """Test VisionProcessor selects CPU when specified.""" + config = VisionConfig(device_preference="cpu") + processor = VisionProcessor(config) + assert processor.device == "cpu" + + +def test_vision_processor_device_selection_cuda_unavailable(mock_torch: Any) -> None: + """Test VisionProcessor falls back to CPU when CUDA unavailable.""" + mock_torch.cuda.is_available.return_value = False + config = VisionConfig(device_preference="cuda") + processor = VisionProcessor(config) + assert processor.device == "cpu" + + +def test_vision_processor_device_selection_cuda_available(mock_torch: Any) -> None: + """Test VisionProcessor selects CUDA when available.""" + mock_torch.cuda.is_available.return_value = True + config = VisionConfig(device_preference="cuda") + processor = VisionProcessor(config) + assert processor.device == "cuda" + + +def test_vision_processor_device_selection_mps_available(mock_torch: Any) -> None: + """Test VisionProcessor selects MPS when available on Apple Silicon.""" + mock_torch.backends.mps.is_available.return_value = True + config = VisionConfig(device_preference="mps") + processor = VisionProcessor(config) + assert processor.device == "mps" + + +def test_vision_processor_device_selection_auto_prefers_mps(mock_torch: Any) -> None: + """Test VisionProcessor auto mode prefers MPS on Apple Silicon.""" + mock_torch.backends.mps.is_available.return_value = True + mock_torch.cuda.is_available.return_value = False + config = VisionConfig(device_preference="auto") + processor = VisionProcessor(config) + assert processor.device == "mps" + + +def test_vision_processor_device_selection_auto_prefers_cuda_over_cpu(mock_torch: Any) -> None: + """Test VisionProcessor auto mode prefers CUDA over CPU.""" + mock_torch.backends.mps.is_available.return_value = False + mock_torch.cuda.is_available.return_value = True + config = VisionConfig(device_preference="auto") + processor = VisionProcessor(config) + assert processor.device == "cuda" + + +def test_vision_processor_initialization(mock_torch: Any, mock_transformers: Any) -> None: + """Test VisionProcessor initializes successfully.""" + config = VisionConfig(model_path="test/model") + processor = VisionProcessor(config) + + assert not processor._initialized + result = processor.initialize() + + assert result is True + assert processor._initialized + mock_transformers["processor"].from_pretrained.assert_called_once_with("test/model") + mock_transformers["model"].from_pretrained.assert_called_once() + + +def test_vision_processor_initialization_failure(mock_torch: Any) -> None: + """Test VisionProcessor handles initialization failure gracefully.""" + with patch("reachy_mini_receptionist.vision.processors.AutoProcessor") as mock_proc: + mock_proc.from_pretrained.side_effect = Exception("Model not found") + + config = VisionConfig(model_path="invalid/model") + processor = VisionProcessor(config) + result = processor.initialize() + + assert result is False + assert not processor._initialized + + +def test_vision_processor_process_image_not_initialized(mock_torch: Any) -> None: + """Test process_image returns error when model not initialized.""" + processor = VisionProcessor() + test_image = np.zeros((480, 640, 3), dtype=np.uint8) + + result = processor.process_image(test_image) + assert result == "Vision model not initialized" + + +def test_vision_processor_process_image_success(mock_torch: Any, mock_transformers: Any) -> None: + """Test process_image processes an image successfully.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + # Mock cv2.imencode to return success + mock_cv2.imencode.return_value = (True, np.array([1, 2, 3], dtype=np.uint8)) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + processor = VisionProcessor() + processor.initialize() + + test_image = np.zeros((480, 640, 3), dtype=np.uint8) + result = processor.process_image(test_image, "Describe this image.") + + assert isinstance(result, str) + assert result == "This is a test description." + + +def test_vision_processor_process_image_encode_failure(mock_torch: Any, mock_transformers: Any) -> None: + """Test process_image handles image encoding failure.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.return_value = (False, None) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + processor = VisionProcessor() + processor.initialize() + + test_image = np.zeros((480, 640, 3), dtype=np.uint8) + result = processor.process_image(test_image) + + assert result == "Failed to encode image" + + +def test_vision_processor_process_image_with_retry(mock_torch: Any, mock_transformers: Any) -> None: + """Test process_image retries on failure.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.return_value = (True, np.array([1, 2, 3], dtype=np.uint8)) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + # Set up the OutOfMemoryError to be a proper exception + mock_torch.cuda.OutOfMemoryError = type("OutOfMemoryError", (Exception,), {}) + + processor = VisionProcessor(VisionConfig(max_retries=3, retry_delay=0.01)) + processor.initialize() + + # Make the model generate fail twice, then succeed + call_count = [0] + assert processor.model is not None + original_generate = processor.model.generate + + def failing_generate(*args: Any, **kwargs: Any) -> Any: + call_count[0] += 1 + if call_count[0] < 3: + raise Exception("Temporary failure") + return original_generate(*args, **kwargs) + + processor.model.generate = failing_generate + + test_image = np.zeros((480, 640, 3), dtype=np.uint8) + result = processor.process_image(test_image) + + assert isinstance(result, str) + assert call_count[0] == 3 + + +def test_vision_processor_extract_response_variants() -> None: + """Test _extract_response handles different response formats.""" + processor = VisionProcessor() + + # Test with "assistant\n" marker + result = processor._extract_response("user prompt\nassistant\nThe response text") + assert result == "The response text" + + # Test with "Assistant:" marker + result = processor._extract_response("User: prompt\nAssistant: Another response") + assert result == "Another response" + + # Test fallback to full text + result = processor._extract_response("Just some text without markers") + assert result == "Just some text without markers" + + +def test_vision_processor_get_model_info(mock_torch: Any, mock_transformers: Any) -> None: + """Test get_model_info returns correct information.""" + mock_torch.cuda.is_available.return_value = True + mock_torch.cuda.get_device_properties.return_value.total_memory = 8 * 1024**3 + + processor = VisionProcessor(VisionConfig(model_path="test/model", device_preference="cpu")) + processor.initialize() + + info = processor.get_model_info() + + assert info["initialized"] is True + assert info["device"] == "cpu" + assert info["model_path"] == "test/model" + assert "cuda_available" in info + + +@pytest.fixture +def mock_camera() -> Mock: + """Create a mock camera object.""" + camera = Mock() + camera.get_latest_frame.return_value = np.zeros((480, 640, 3), dtype=np.uint8) + return camera + + +def test_vision_manager_initialization(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager initializes successfully.""" + config = VisionConfig(vision_interval=2.0) + manager = VisionManager(mock_camera, config) + + assert manager.vision_interval == 2.0 + assert manager.processor._initialized + + +def test_vision_manager_initialization_failure(mock_torch: Any, mock_camera: Mock) -> None: + """Test VisionManager raises error when processor initialization fails.""" + with patch("reachy_mini_receptionist.vision.processors.AutoProcessor") as mock_proc: + mock_proc.from_pretrained.side_effect = Exception("Model not found") + + with pytest.raises(RuntimeError, match="Vision processor initialization failed"): + VisionManager(mock_camera, VisionConfig()) + + +def test_vision_manager_start_stop(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager can start and stop.""" + manager = VisionManager(mock_camera, VisionConfig()) + + manager.start() + assert manager._thread is not None + assert manager._thread.is_alive() + assert not manager._stop_event.is_set() + + time.sleep(0.1) # Let thread run briefly + + manager.stop() + assert manager._stop_event.is_set() + assert not manager._thread.is_alive() + + +def test_vision_manager_processes_frames(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager processes frames at intervals.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.return_value = (True, np.array([1, 2, 3], dtype=np.uint8)) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + config = VisionConfig(vision_interval=0.1) # Fast interval for testing + manager = VisionManager(mock_camera, config) + + manager.start() + time.sleep(0.3) # Wait for at least 2 processing cycles + manager.stop() + + # Camera should have been called at least once + assert mock_camera.get_latest_frame.call_count >= 1 + + +def test_vision_manager_handles_none_frame(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager handles None frame gracefully.""" + mock_camera.get_latest_frame.return_value = None + + config = VisionConfig(vision_interval=0.1) + manager = VisionManager(mock_camera, config) + + manager.start() + time.sleep(0.2) + manager.stop() + + # Verify camera was called but no crashes occurred + assert mock_camera.get_latest_frame.called + + +def test_vision_manager_handles_processing_error(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager handles processing errors gracefully.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.side_effect = Exception("Processing error") + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + config = VisionConfig(vision_interval=0.1) + manager = VisionManager(mock_camera, config) + + manager.start() + time.sleep(0.2) + manager.stop() + + # Verify thread stopped gracefully despite errors + assert manager._stop_event.is_set() + + +def test_vision_manager_get_status(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager get_status returns correct information.""" + manager = VisionManager(mock_camera, VisionConfig(vision_interval=5.0)) + + status = manager.get_status() + + assert "last_processed" in status + assert "processor_info" in status + assert "config" in status + assert status["config"]["interval"] == 5.0 + + +def test_vision_manager_skips_invalid_responses(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager doesn't update timestamp for invalid responses.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.return_value = (True, np.array([1, 2, 3], dtype=np.uint8)) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + # Make processor return invalid response + config = VisionConfig(vision_interval=0.1) + manager = VisionManager(mock_camera, config) + + # Mock the processor's process_image method to return invalid response + with patch.object(manager.processor, 'process_image', return_value="Vision model not initialized"): + initial_time = manager._last_processed_time + + manager.start() + time.sleep(0.2) + manager.stop() + + # Last processed time should not have been updated + assert manager._last_processed_time == initial_time + + +def test_initialize_vision_manager_success(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test initialize_vision_manager creates VisionManager successfully.""" + with patch("reachy_mini_receptionist.vision.processors.snapshot_download") as mock_download, \ + patch("reachy_mini_receptionist.vision.processors.os.makedirs"), \ + patch("reachy_mini_receptionist.vision.processors.config") as mock_config: + + mock_config.LOCAL_VISION_MODEL = "test/model" + mock_config.HF_HOME = "/tmp/hf_cache" + + result = initialize_vision_manager(mock_camera) + + assert result is not None + assert isinstance(result, VisionManager) + mock_download.assert_called_once() + + +def test_initialize_vision_manager_download_failure(mock_torch: Any, mock_camera: Mock) -> None: + """Test initialize_vision_manager handles download failure.""" + with patch("reachy_mini_receptionist.vision.processors.snapshot_download") as mock_download, \ + patch("reachy_mini_receptionist.vision.processors.os.makedirs"), \ + patch("reachy_mini_receptionist.vision.processors.config") as mock_config: + + mock_config.LOCAL_VISION_MODEL = "test/model" + mock_config.HF_HOME = "/tmp/hf_cache" + mock_download.side_effect = Exception("Network error") + + result = initialize_vision_manager(mock_camera) + + assert result is None + + +def test_initialize_vision_manager_processor_failure(mock_torch: Any, mock_camera: Mock) -> None: + """Test initialize_vision_manager handles processor initialization failure.""" + with patch("reachy_mini_receptionist.vision.processors.snapshot_download"), \ + patch("reachy_mini_receptionist.vision.processors.os.makedirs"), \ + patch("reachy_mini_receptionist.vision.processors.config") as mock_config, \ + patch("reachy_mini_receptionist.vision.processors.AutoProcessor") as mock_proc: + + mock_config.LOCAL_VISION_MODEL = "test/model" + mock_config.HF_HOME = "/tmp/hf_cache" + mock_proc.from_pretrained.side_effect = Exception("Model load error") + + result = initialize_vision_manager(mock_camera) + + assert result is None + + +def test_vision_processor_cuda_oom_recovery(mock_torch: Any, mock_transformers: Any) -> None: + """Test VisionProcessor recovers from CUDA OOM errors.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.return_value = (True, np.array([1, 2, 3], dtype=np.uint8)) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + processor = VisionProcessor(VisionConfig(max_retries=2, retry_delay=0.01)) + processor.initialize() + processor.device = "cuda" # Force CUDA for this test + + # Make generate raise OOM error + mock_torch.cuda.OutOfMemoryError = type("OutOfMemoryError", (Exception,), {}) + assert processor.model is not None + processor.model.generate.side_effect = mock_torch.cuda.OutOfMemoryError("OOM") + + test_image = np.zeros((480, 640, 3), dtype=np.uint8) + result = processor.process_image(test_image) + + assert "GPU out of memory" in result + mock_torch.cuda.empty_cache.assert_called() + + +def test_vision_processor_cache_cleanup_mps(mock_torch: Any, mock_transformers: Any) -> None: + """Test VisionProcessor cleans up MPS cache after processing.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.return_value = (True, np.array([1, 2, 3], dtype=np.uint8)) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + processor = VisionProcessor() + processor.initialize() + processor.device = "mps" # Force MPS for this test + + test_image = np.zeros((480, 640, 3), dtype=np.uint8) + processor.process_image(test_image) + + # Should call mps empty_cache + mock_torch.mps.empty_cache.assert_called() + + +def test_vision_manager_thread_safety(mock_torch: Any, mock_transformers: Any, mock_camera: Mock) -> None: + """Test VisionManager thread safety with multiple start/stop cycles.""" + with patch("reachy_mini_receptionist.vision.processors.cv2") as mock_cv2: + mock_cv2.imencode.return_value = (True, np.array([1, 2, 3], dtype=np.uint8)) + mock_cv2.IMWRITE_JPEG_QUALITY = 1 + + config = VisionConfig(vision_interval=0.05) + manager = VisionManager(mock_camera, config) + + # Multiple start/stop cycles + for _ in range(3): + manager.start() + time.sleep(0.1) + manager.stop() + time.sleep(0.05) + + # Should not crash or leave dangling threads + assert manager._stop_event.is_set()