tomrikert commited on
Commit ·
c1956d8
0
Parent(s):
Initial release of ClawBody
Browse filesGive your OpenClaw AI agent a physical robot body with Reachy Mini.
Features:
- Real-time voice conversation via OpenAI Realtime API
- OpenClaw/Clawson intelligence for AI responses
- Vision through robot camera
- Expressive movements (head, emotions, dances)
- .env.example +39 -0
- .gitignore +62 -0
- CONTRIBUTING.md +78 -0
- LICENSE +191 -0
- README.md +212 -0
- index.html +129 -0
- openclaw-skill/SKILL.md +97 -0
- pyproject.toml +104 -0
- src/reachy_mini_openclaw/__init__.py +9 -0
- src/reachy_mini_openclaw/audio/__init__.py +5 -0
- src/reachy_mini_openclaw/audio/head_wobbler.py +223 -0
- src/reachy_mini_openclaw/config.py +59 -0
- src/reachy_mini_openclaw/gradio_app.py +194 -0
- src/reachy_mini_openclaw/main.py +420 -0
- src/reachy_mini_openclaw/moves.py +551 -0
- src/reachy_mini_openclaw/openai_realtime.py +428 -0
- src/reachy_mini_openclaw/openclaw_bridge.py +286 -0
- src/reachy_mini_openclaw/prompts.py +98 -0
- src/reachy_mini_openclaw/prompts/default.txt +35 -0
- src/reachy_mini_openclaw/tools/__init__.py +17 -0
- src/reachy_mini_openclaw/tools/core_tools.py +322 -0
- style.css +284 -0
.env.example
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ClawBody Configuration
|
| 2 |
+
# Give your OpenClaw AI agent a physical robot body!
|
| 3 |
+
|
| 4 |
+
# ==============================================================================
|
| 5 |
+
# REQUIRED: OpenAI API Key
|
| 6 |
+
# ==============================================================================
|
| 7 |
+
# Get your key at: https://platform.openai.com/api-keys
|
| 8 |
+
# Requires Realtime API access
|
| 9 |
+
OPENAI_API_KEY=sk-your-openai-key
|
| 10 |
+
|
| 11 |
+
# ==============================================================================
|
| 12 |
+
# REQUIRED: OpenClaw Gateway
|
| 13 |
+
# ==============================================================================
|
| 14 |
+
# The URL where your OpenClaw gateway is running
|
| 15 |
+
# If running on the same machine as the robot, use the host machine's IP
|
| 16 |
+
OPENCLAW_GATEWAY_URL=http://192.168.1.100:18789
|
| 17 |
+
|
| 18 |
+
# Your OpenClaw gateway authentication token
|
| 19 |
+
# Find this in ~/.openclaw/openclaw.json under gateway.token
|
| 20 |
+
OPENCLAW_TOKEN=your-gateway-token
|
| 21 |
+
|
| 22 |
+
# Agent ID to use (default: main)
|
| 23 |
+
OPENCLAW_AGENT_ID=main
|
| 24 |
+
|
| 25 |
+
# ==============================================================================
|
| 26 |
+
# OPTIONAL: Voice Settings
|
| 27 |
+
# ==============================================================================
|
| 28 |
+
# OpenAI Realtime voice (alloy, echo, fable, onyx, nova, shimmer, cedar)
|
| 29 |
+
OPENAI_VOICE=cedar
|
| 30 |
+
|
| 31 |
+
# OpenAI model for Realtime API
|
| 32 |
+
OPENAI_MODEL=gpt-4o-realtime-preview-2024-12-17
|
| 33 |
+
|
| 34 |
+
# ==============================================================================
|
| 35 |
+
# OPTIONAL: Features
|
| 36 |
+
# ==============================================================================
|
| 37 |
+
# Enable/disable features (true/false)
|
| 38 |
+
ENABLE_CAMERA=true
|
| 39 |
+
ENABLE_OPENCLAW_TOOLS=true
|
.gitignore
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ClawBody .gitignore
|
| 2 |
+
|
| 3 |
+
# Environment and secrets
|
| 4 |
+
.env
|
| 5 |
+
*.env.local
|
| 6 |
+
|
| 7 |
+
# Python
|
| 8 |
+
__pycache__/
|
| 9 |
+
*.py[cod]
|
| 10 |
+
*$py.class
|
| 11 |
+
*.so
|
| 12 |
+
.Python
|
| 13 |
+
build/
|
| 14 |
+
develop-eggs/
|
| 15 |
+
dist/
|
| 16 |
+
downloads/
|
| 17 |
+
eggs/
|
| 18 |
+
.eggs/
|
| 19 |
+
lib/
|
| 20 |
+
lib64/
|
| 21 |
+
parts/
|
| 22 |
+
sdist/
|
| 23 |
+
var/
|
| 24 |
+
wheels/
|
| 25 |
+
*.egg-info/
|
| 26 |
+
.installed.cfg
|
| 27 |
+
*.egg
|
| 28 |
+
|
| 29 |
+
# Virtual environments
|
| 30 |
+
.venv/
|
| 31 |
+
venv/
|
| 32 |
+
ENV/
|
| 33 |
+
env/
|
| 34 |
+
|
| 35 |
+
# IDE
|
| 36 |
+
.idea/
|
| 37 |
+
.vscode/
|
| 38 |
+
*.swp
|
| 39 |
+
*.swo
|
| 40 |
+
*~
|
| 41 |
+
|
| 42 |
+
# Testing
|
| 43 |
+
.pytest_cache/
|
| 44 |
+
.coverage
|
| 45 |
+
htmlcov/
|
| 46 |
+
.tox/
|
| 47 |
+
.nox/
|
| 48 |
+
|
| 49 |
+
# Type checking
|
| 50 |
+
.mypy_cache/
|
| 51 |
+
.dmypy.json
|
| 52 |
+
dmypy.json
|
| 53 |
+
|
| 54 |
+
# Logs
|
| 55 |
+
*.log
|
| 56 |
+
|
| 57 |
+
# OS
|
| 58 |
+
.DS_Store
|
| 59 |
+
Thumbs.db
|
| 60 |
+
|
| 61 |
+
# Package manager
|
| 62 |
+
uv.lock
|
CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Contributing to Reachy Mini OpenClaw
|
| 2 |
+
|
| 3 |
+
Thank you for your interest in contributing! This project welcomes contributions from the community.
|
| 4 |
+
|
| 5 |
+
## How to Contribute
|
| 6 |
+
|
| 7 |
+
### Reporting Bugs
|
| 8 |
+
|
| 9 |
+
If you find a bug, please open an issue with:
|
| 10 |
+
- A clear title and description
|
| 11 |
+
- Steps to reproduce the issue
|
| 12 |
+
- Expected vs actual behavior
|
| 13 |
+
- Your environment (OS, Python version, robot model)
|
| 14 |
+
|
| 15 |
+
### Suggesting Features
|
| 16 |
+
|
| 17 |
+
Feature requests are welcome! Please open an issue with:
|
| 18 |
+
- A clear description of the feature
|
| 19 |
+
- Use cases and motivation
|
| 20 |
+
- Any technical considerations
|
| 21 |
+
|
| 22 |
+
### Pull Requests
|
| 23 |
+
|
| 24 |
+
1. Fork the repository
|
| 25 |
+
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
|
| 26 |
+
3. Make your changes
|
| 27 |
+
4. Add tests if applicable
|
| 28 |
+
5. Run linting: `ruff check . && ruff format .`
|
| 29 |
+
6. Commit your changes (`git commit -m 'Add amazing feature'`)
|
| 30 |
+
7. Push to the branch (`git push origin feature/amazing-feature`)
|
| 31 |
+
8. Open a Pull Request
|
| 32 |
+
|
| 33 |
+
## Development Setup
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
# Clone your fork
|
| 37 |
+
git clone https://github.com/YOUR_USERNAME/reachy_mini_openclaw.git
|
| 38 |
+
cd reachy_mini_openclaw
|
| 39 |
+
|
| 40 |
+
# Install in development mode
|
| 41 |
+
pip install -e ".[dev]"
|
| 42 |
+
|
| 43 |
+
# Run tests
|
| 44 |
+
pytest
|
| 45 |
+
|
| 46 |
+
# Format code
|
| 47 |
+
ruff check --fix .
|
| 48 |
+
ruff format .
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
## Code Style
|
| 52 |
+
|
| 53 |
+
- Follow PEP 8
|
| 54 |
+
- Use type hints
|
| 55 |
+
- Write docstrings for public functions and classes
|
| 56 |
+
- Keep functions focused and small
|
| 57 |
+
|
| 58 |
+
## Where to Submit Contributions
|
| 59 |
+
|
| 60 |
+
### This Project
|
| 61 |
+
Submit PRs directly to this repository for:
|
| 62 |
+
- Bug fixes
|
| 63 |
+
- New features
|
| 64 |
+
- Documentation improvements
|
| 65 |
+
- New personality profiles
|
| 66 |
+
|
| 67 |
+
### Reachy Mini Ecosystem
|
| 68 |
+
- **SDK improvements**: [pollen-robotics/reachy_mini](https://github.com/pollen-robotics/reachy_mini)
|
| 69 |
+
- **New dances/emotions**: [reachy_mini_dances_library](https://github.com/pollen-robotics/reachy_mini_dances_library)
|
| 70 |
+
- **Apps for the app store**: Submit to [Hugging Face Spaces](https://huggingface.co/spaces)
|
| 71 |
+
|
| 72 |
+
### OpenClaw Ecosystem
|
| 73 |
+
- **New skills**: Submit to [MoltDirectory](https://github.com/neonone123/moltdirectory)
|
| 74 |
+
- **Core OpenClaw**: [openclaw/openclaw](https://github.com/openclaw/openclaw)
|
| 75 |
+
|
| 76 |
+
## License
|
| 77 |
+
|
| 78 |
+
By contributing, you agree that your contributions will be licensed under the Apache 2.0 License.
|
LICENSE
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to the Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no theory of
|
| 154 |
+
liability, whether in contract, strict liability, or tort
|
| 155 |
+
(including negligence or otherwise) arising in any way out of
|
| 156 |
+
the use or inability to use the Work (even if such Holder or
|
| 157 |
+
other party has been advised of the possibility of such damages),
|
| 158 |
+
shall any Contributor be liable to You for damages, including any
|
| 159 |
+
direct, indirect, special, incidental, or consequential damages of
|
| 160 |
+
any character arising as a result of this License or out of the use
|
| 161 |
+
or inability to use the Work (including but not limited to damages
|
| 162 |
+
for loss of goodwill, work stoppage, computer failure or malfunction,
|
| 163 |
+
or any and all other commercial damages or losses), even if such
|
| 164 |
+
Contributor has been advised of the possibility of such damages.
|
| 165 |
+
|
| 166 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 167 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 168 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 169 |
+
or other liability obligations and/or rights consistent with this
|
| 170 |
+
License. However, in accepting such obligations, You may act only
|
| 171 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 172 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 173 |
+
defend, and hold each Contributor harmless for any liability
|
| 174 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 175 |
+
of your accepting any such warranty or additional liability.
|
| 176 |
+
|
| 177 |
+
END OF TERMS AND CONDITIONS
|
| 178 |
+
|
| 179 |
+
Copyright 2024 Tom
|
| 180 |
+
|
| 181 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 182 |
+
you may not use this file except in compliance with the License.
|
| 183 |
+
You may obtain a copy of the License at
|
| 184 |
+
|
| 185 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 186 |
+
|
| 187 |
+
Unless required by applicable law or agreed to in writing, software
|
| 188 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 189 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 190 |
+
See the License for the specific language governing permissions and
|
| 191 |
+
limitations under the License.
|
README.md
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: ClawBody
|
| 3 |
+
emoji: 🦞
|
| 4 |
+
colorFrom: red
|
| 5 |
+
colorTo: purple
|
| 6 |
+
sdk: static
|
| 7 |
+
pinned: false
|
| 8 |
+
short_description: Give your OpenClaw AI a physical robot body
|
| 9 |
+
tags:
|
| 10 |
+
- reachy_mini
|
| 11 |
+
- reachy_mini_python_app
|
| 12 |
+
- openclaw
|
| 13 |
+
- clawson
|
| 14 |
+
- embodied-ai
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# 🦞🤖 ClawBody
|
| 18 |
+
|
| 19 |
+
**Give your OpenClaw AI agent a physical robot body!**
|
| 20 |
+
|
| 21 |
+
ClawBody combines OpenClaw's AI intelligence with Reachy Mini's expressive robot body, using OpenAI's Realtime API for ultra-responsive voice conversation. Your OpenClaw assistant (Clawson) can now see, hear, speak, and move in the physical world.
|
| 22 |
+
|
| 23 |
+
[](LICENSE)
|
| 24 |
+
[](https://www.python.org/downloads/)
|
| 25 |
+
|
| 26 |
+
## ✨ Features
|
| 27 |
+
|
| 28 |
+
- **🎤 Real-time Voice Conversation**: OpenAI Realtime API for sub-second response latency
|
| 29 |
+
- **🧠 OpenClaw Intelligence**: Your responses come from OpenClaw with full tool access
|
| 30 |
+
- **👀 Vision**: See through the robot's camera and describe the environment
|
| 31 |
+
- **💃 Expressive Movements**: Natural head movements, emotions, dances, and audio-driven wobble
|
| 32 |
+
- **🦞 Clawson Embodied**: Your friendly space lobster AI assistant, now with a body!
|
| 33 |
+
|
| 34 |
+
## 🏗️ Architecture
|
| 35 |
+
|
| 36 |
+
```
|
| 37 |
+
┌─────────────────────────────────────────────────────────────────┐
|
| 38 |
+
│ Your Voice / Microphone │
|
| 39 |
+
└─────────────────────────────┬───────────────────────────────────┘
|
| 40 |
+
│
|
| 41 |
+
▼
|
| 42 |
+
┌─────────────────────────────────────────────────────────────────┐
|
| 43 |
+
│ Reachy Mini Robot │
|
| 44 |
+
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────────┐ │
|
| 45 |
+
│ │ Microphone │ │ Camera │ │ Movement System │ │
|
| 46 |
+
│ │ (input) │ │ (vision) │ │ (head, antennas, body) │ │
|
| 47 |
+
│ └──────┬──────┘ └──────┬──────┘ └────────────▲────────────┘ │
|
| 48 |
+
└─────────┼────────────────┼──────────────────────┼───────────────┘
|
| 49 |
+
│ │ │
|
| 50 |
+
▼ ▼ │
|
| 51 |
+
┌─────────────────────────────────────────────────┼───────────────┐
|
| 52 |
+
│ ClawBody │ │
|
| 53 |
+
│ ┌─────────────────────────────────────────────┼────────────┐ │
|
| 54 |
+
│ │ OpenAI Realtime API Handler │ │ │
|
| 55 |
+
│ │ • Speech recognition (Whisper) │ │ │
|
| 56 |
+
│ │ • Text-to-speech (voices) ─┘ │ │
|
| 57 |
+
│ │ • Audio analysis → head wobble │ │
|
| 58 |
+
│ └─────────────────────────────────────────────────────────┘ │
|
| 59 |
+
│ │ │
|
| 60 |
+
│ ▼ │
|
| 61 |
+
│ ┌─────────────────────────────────────────────────────────┐ │
|
| 62 |
+
│ │ OpenClaw Gateway Bridge │ │
|
| 63 |
+
│ │ • AI responses from Clawson │ │
|
| 64 |
+
│ │ • Full OpenClaw tool access │ │
|
| 65 |
+
│ │ • Conversation memory & context │ │
|
| 66 |
+
│ └─────────────────────────────────────────────────────────┘ │
|
| 67 |
+
└─────────────────────────────────────────────────────────────────┘
|
| 68 |
+
│
|
| 69 |
+
▼
|
| 70 |
+
┌──────────���──────────────────────────────────────────────────────┐
|
| 71 |
+
│ OpenClaw Gateway │
|
| 72 |
+
│ • Web browsing • Calendar • Smart home • Memory • Tools │
|
| 73 |
+
└─────────────────────────────────────────────────────────────────┘
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
## 📋 Prerequisites
|
| 77 |
+
|
| 78 |
+
### Hardware
|
| 79 |
+
- [Reachy Mini](https://www.pollen-robotics.com/reachy-mini/) robot (Wireless or Lite)
|
| 80 |
+
|
| 81 |
+
### Software
|
| 82 |
+
- Python 3.11+
|
| 83 |
+
- [Reachy Mini SDK](https://github.com/pollen-robotics/reachy_mini) installed
|
| 84 |
+
- [OpenClaw](https://github.com/openclaw/openclaw) gateway running
|
| 85 |
+
- OpenAI API key with Realtime API access
|
| 86 |
+
|
| 87 |
+
## 🚀 Installation
|
| 88 |
+
|
| 89 |
+
### On the Reachy Mini Robot
|
| 90 |
+
|
| 91 |
+
```bash
|
| 92 |
+
# SSH into the robot
|
| 93 |
+
ssh pollen@reachy-mini.local
|
| 94 |
+
|
| 95 |
+
# Clone the repository
|
| 96 |
+
git clone https://github.com/yourusername/clawbody.git
|
| 97 |
+
cd clawbody
|
| 98 |
+
|
| 99 |
+
# Install in the apps virtual environment
|
| 100 |
+
/venvs/apps_venv/bin/pip install -e .
|
| 101 |
+
```
|
| 102 |
+
|
| 103 |
+
### Using pip (Development)
|
| 104 |
+
|
| 105 |
+
```bash
|
| 106 |
+
# Clone the repository
|
| 107 |
+
git clone https://github.com/yourusername/clawbody.git
|
| 108 |
+
cd clawbody
|
| 109 |
+
|
| 110 |
+
# Create virtual environment
|
| 111 |
+
python -m venv .venv
|
| 112 |
+
source .venv/bin/activate
|
| 113 |
+
|
| 114 |
+
# Install
|
| 115 |
+
pip install -e .
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
## ⚙️ Configuration
|
| 119 |
+
|
| 120 |
+
1. Copy the example environment file:
|
| 121 |
+
|
| 122 |
+
```bash
|
| 123 |
+
cp .env.example .env
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
2. Edit `.env` with your configuration:
|
| 127 |
+
|
| 128 |
+
```bash
|
| 129 |
+
# Required
|
| 130 |
+
OPENAI_API_KEY=sk-...your-key...
|
| 131 |
+
|
| 132 |
+
# OpenClaw Gateway (required for AI responses)
|
| 133 |
+
OPENCLAW_GATEWAY_URL=http://your-host-ip:18790
|
| 134 |
+
OPENCLAW_TOKEN=your-gateway-token
|
| 135 |
+
OPENCLAW_AGENT_ID=main
|
| 136 |
+
|
| 137 |
+
# Optional - Customize voice
|
| 138 |
+
OPENAI_VOICE=cedar
|
| 139 |
+
```
|
| 140 |
+
|
| 141 |
+
## 🎮 Usage
|
| 142 |
+
|
| 143 |
+
### Console Mode
|
| 144 |
+
|
| 145 |
+
```bash
|
| 146 |
+
# Basic usage
|
| 147 |
+
clawbody
|
| 148 |
+
|
| 149 |
+
# With debug logging
|
| 150 |
+
clawbody --debug
|
| 151 |
+
|
| 152 |
+
# With specific robot
|
| 153 |
+
clawbody --robot-name my-reachy
|
| 154 |
+
```
|
| 155 |
+
|
| 156 |
+
### Web UI Mode
|
| 157 |
+
|
| 158 |
+
```bash
|
| 159 |
+
# Launch Gradio interface
|
| 160 |
+
clawbody --gradio
|
| 161 |
+
```
|
| 162 |
+
|
| 163 |
+
Then open http://localhost:7860 in your browser.
|
| 164 |
+
|
| 165 |
+
### As a Reachy Mini App
|
| 166 |
+
|
| 167 |
+
ClawBody registers as a Reachy Mini App, so you can launch it from the robot's dashboard after installation.
|
| 168 |
+
|
| 169 |
+
### CLI Options
|
| 170 |
+
|
| 171 |
+
| Option | Description |
|
| 172 |
+
|--------|-------------|
|
| 173 |
+
| `--debug` | Enable debug logging |
|
| 174 |
+
| `--gradio` | Launch web UI instead of console mode |
|
| 175 |
+
| `--robot-name NAME` | Specify robot name for connection |
|
| 176 |
+
| `--gateway-url URL` | OpenClaw gateway URL |
|
| 177 |
+
| `--no-camera` | Disable camera functionality |
|
| 178 |
+
| `--no-openclaw` | Disable OpenClaw integration |
|
| 179 |
+
|
| 180 |
+
## 🛠️ Robot Capabilities
|
| 181 |
+
|
| 182 |
+
ClawBody gives Clawson these physical abilities:
|
| 183 |
+
|
| 184 |
+
| Capability | Description |
|
| 185 |
+
|------------|-------------|
|
| 186 |
+
| **Look** | Move head to look in directions (left, right, up, down) |
|
| 187 |
+
| **See** | Capture images through the robot's camera |
|
| 188 |
+
| **Dance** | Perform expressive dance animations |
|
| 189 |
+
| **Emotions** | Express emotions through movement (happy, curious, thinking, etc.) |
|
| 190 |
+
| **Speak** | Voice output through the robot's speaker |
|
| 191 |
+
| **Listen** | Hear through the robot's microphone |
|
| 192 |
+
|
| 193 |
+
## 📄 License
|
| 194 |
+
|
| 195 |
+
This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.
|
| 196 |
+
|
| 197 |
+
## 🙏 Acknowledgments
|
| 198 |
+
|
| 199 |
+
ClawBody builds on:
|
| 200 |
+
|
| 201 |
+
- [Pollen Robotics](https://www.pollen-robotics.com/) - Reachy Mini robot and SDK
|
| 202 |
+
- [OpenClaw](https://github.com/openclaw/openclaw) - AI assistant framework (Clawson!)
|
| 203 |
+
- [OpenAI](https://openai.com/) - Realtime API for voice I/O
|
| 204 |
+
- [pollen-robotics/reachy_mini_conversation_app](https://huggingface.co/spaces/pollen-robotics/reachy_mini_conversation_app) - Movement and audio systems
|
| 205 |
+
|
| 206 |
+
## 🤝 Contributing
|
| 207 |
+
|
| 208 |
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
| 209 |
+
|
| 210 |
+
- **This project**: [GitHub Issues](https://github.com/yourusername/clawbody/issues)
|
| 211 |
+
- **OpenClaw Skills**: Submit ClawBody as a skill to [ClawHub](https://docs.openclaw.ai/tools/clawhub)
|
| 212 |
+
- **Reachy Mini Apps**: Submit to [Pollen Robotics](https://github.com/pollen-robotics)
|
index.html
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 6 |
+
<title>ClawBody - OpenClaw + Reachy Mini</title>
|
| 7 |
+
<link rel="stylesheet" href="style.css">
|
| 8 |
+
</head>
|
| 9 |
+
<body>
|
| 10 |
+
<div class="container">
|
| 11 |
+
<header>
|
| 12 |
+
<div class="logo">🦞🤖</div>
|
| 13 |
+
<h1>ClawBody</h1>
|
| 14 |
+
<p class="tagline">Give your OpenClaw AI agent a physical robot body</p>
|
| 15 |
+
</header>
|
| 16 |
+
|
| 17 |
+
<main>
|
| 18 |
+
<section class="hero">
|
| 19 |
+
<div class="hero-content">
|
| 20 |
+
<h2>Clawson, Embodied</h2>
|
| 21 |
+
<p>
|
| 22 |
+
ClawBody bridges <a href="https://github.com/openclaw/openclaw">OpenClaw</a>
|
| 23 |
+
with <a href="https://github.com/pollen-robotics/reachy_mini">Reachy Mini</a>,
|
| 24 |
+
letting your AI assistant see, hear, speak, and move in the physical world.
|
| 25 |
+
</p>
|
| 26 |
+
</div>
|
| 27 |
+
</section>
|
| 28 |
+
|
| 29 |
+
<section class="features">
|
| 30 |
+
<h3>Features</h3>
|
| 31 |
+
<div class="feature-grid">
|
| 32 |
+
<div class="feature">
|
| 33 |
+
<span class="feature-icon">🎤</span>
|
| 34 |
+
<h4>Real-time Voice</h4>
|
| 35 |
+
<p>OpenAI Realtime API for sub-second voice interaction</p>
|
| 36 |
+
</div>
|
| 37 |
+
<div class="feature">
|
| 38 |
+
<span class="feature-icon">🧠</span>
|
| 39 |
+
<h4>OpenClaw Intelligence</h4>
|
| 40 |
+
<p>Full Clawson capabilities - tools, memory, personality</p>
|
| 41 |
+
</div>
|
| 42 |
+
<div class="feature">
|
| 43 |
+
<span class="feature-icon">👀</span>
|
| 44 |
+
<h4>Vision</h4>
|
| 45 |
+
<p>See through the robot's camera and describe the world</p>
|
| 46 |
+
</div>
|
| 47 |
+
<div class="feature">
|
| 48 |
+
<span class="feature-icon">💃</span>
|
| 49 |
+
<h4>Expressive Movement</h4>
|
| 50 |
+
<p>Natural head movements, emotions, dances, and wobble</p>
|
| 51 |
+
</div>
|
| 52 |
+
</div>
|
| 53 |
+
</section>
|
| 54 |
+
|
| 55 |
+
<section class="architecture">
|
| 56 |
+
<h3>How It Works</h3>
|
| 57 |
+
<div class="arch-diagram">
|
| 58 |
+
<div class="arch-flow">
|
| 59 |
+
<div class="arch-box user">🗣️ You speak</div>
|
| 60 |
+
<div class="arch-arrow">→</div>
|
| 61 |
+
<div class="arch-box robot">🤖 Reachy Mini<br><small>microphone + camera</small></div>
|
| 62 |
+
<div class="arch-arrow">→</div>
|
| 63 |
+
<div class="arch-box clawbody">🦞 ClawBody<br><small>OpenAI Realtime</small></div>
|
| 64 |
+
<div class="arch-arrow">→</div>
|
| 65 |
+
<div class="arch-box openclaw">🧠 OpenClaw<br><small>Clawson responds</small></div>
|
| 66 |
+
</div>
|
| 67 |
+
<div class="arch-return">
|
| 68 |
+
<div class="arch-arrow-return">← Robot speaks & moves ←</div>
|
| 69 |
+
</div>
|
| 70 |
+
</div>
|
| 71 |
+
</section>
|
| 72 |
+
|
| 73 |
+
<section class="installation">
|
| 74 |
+
<h3>Quick Start</h3>
|
| 75 |
+
<div class="code-block">
|
| 76 |
+
<pre><code># SSH into your Reachy Mini
|
| 77 |
+
ssh pollen@reachy-mini.local
|
| 78 |
+
|
| 79 |
+
# Clone and install
|
| 80 |
+
git clone https://huggingface.co/spaces/YOUR_USERNAME/clawbody
|
| 81 |
+
cd clawbody
|
| 82 |
+
pip install -e .
|
| 83 |
+
|
| 84 |
+
# Configure
|
| 85 |
+
cp .env.example .env
|
| 86 |
+
# Edit .env with your OpenAI key and OpenClaw gateway URL
|
| 87 |
+
|
| 88 |
+
# Run
|
| 89 |
+
clawbody</code></pre>
|
| 90 |
+
</div>
|
| 91 |
+
</section>
|
| 92 |
+
|
| 93 |
+
<section class="requirements">
|
| 94 |
+
<h3>Requirements</h3>
|
| 95 |
+
<ul>
|
| 96 |
+
<li><strong>Reachy Mini</strong> robot (Wireless or Lite)</li>
|
| 97 |
+
<li><strong>OpenClaw</strong> gateway running on your network</li>
|
| 98 |
+
<li><strong>OpenAI API key</strong> with Realtime API access</li>
|
| 99 |
+
</ul>
|
| 100 |
+
</section>
|
| 101 |
+
|
| 102 |
+
<section class="links">
|
| 103 |
+
<h3>Resources</h3>
|
| 104 |
+
<div class="link-grid">
|
| 105 |
+
<a href="https://github.com/openclaw/openclaw" class="link-card">
|
| 106 |
+
<span>🦞</span>
|
| 107 |
+
<span>OpenClaw</span>
|
| 108 |
+
</a>
|
| 109 |
+
<a href="https://github.com/pollen-robotics/reachy_mini" class="link-card">
|
| 110 |
+
<span>🤖</span>
|
| 111 |
+
<span>Reachy Mini SDK</span>
|
| 112 |
+
</a>
|
| 113 |
+
<a href="https://platform.openai.com/docs/guides/realtime" class="link-card">
|
| 114 |
+
<span>🎙️</span>
|
| 115 |
+
<span>OpenAI Realtime</span>
|
| 116 |
+
</a>
|
| 117 |
+
</div>
|
| 118 |
+
</section>
|
| 119 |
+
</main>
|
| 120 |
+
|
| 121 |
+
<footer>
|
| 122 |
+
<p>
|
| 123 |
+
Made with 🦞 by <a href="https://github.com/yourusername">Tom</a>
|
| 124 |
+
· Apache 2.0 License
|
| 125 |
+
</p>
|
| 126 |
+
</footer>
|
| 127 |
+
</div>
|
| 128 |
+
</body>
|
| 129 |
+
</html>
|
openclaw-skill/SKILL.md
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reachy Mini Robot Body
|
| 2 |
+
|
| 3 |
+
Give your OpenClaw agent a physical presence with Reachy Mini robot.
|
| 4 |
+
|
| 5 |
+
## Description
|
| 6 |
+
|
| 7 |
+
This skill enables OpenClaw to embody a Reachy Mini robot, allowing your AI assistant to:
|
| 8 |
+
|
| 9 |
+
- **See**: View the world through the robot's camera
|
| 10 |
+
- **Hear**: Listen to conversations via the robot's microphone
|
| 11 |
+
- **Speak**: Respond with natural voice through the robot's speaker
|
| 12 |
+
- **Move**: Express emotions and reactions through expressive head movements
|
| 13 |
+
|
| 14 |
+
Using OpenAI's Realtime API, the robot responds with sub-second latency for natural conversation flow.
|
| 15 |
+
|
| 16 |
+
## Requirements
|
| 17 |
+
|
| 18 |
+
### Hardware
|
| 19 |
+
- [Reachy Mini](https://www.hf.co/reachy-mini/) robot (Wireless or Lite version)
|
| 20 |
+
- The robot must be powered on and reachable on the network
|
| 21 |
+
|
| 22 |
+
### Software
|
| 23 |
+
- Python 3.11+
|
| 24 |
+
- OpenAI API key with Realtime API access
|
| 25 |
+
- OpenClaw gateway running (for extended capabilities)
|
| 26 |
+
|
| 27 |
+
## Installation
|
| 28 |
+
|
| 29 |
+
```bash
|
| 30 |
+
# Install the package
|
| 31 |
+
pip install reachy-mini-openclaw
|
| 32 |
+
|
| 33 |
+
# Or from source
|
| 34 |
+
git clone https://github.com/yourusername/reachy_mini_openclaw.git
|
| 35 |
+
cd reachy_mini_openclaw
|
| 36 |
+
pip install -e .
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
## Configuration
|
| 40 |
+
|
| 41 |
+
Create a `.env` file with your OpenAI API key:
|
| 42 |
+
|
| 43 |
+
```bash
|
| 44 |
+
OPENAI_API_KEY=sk-your-key-here
|
| 45 |
+
OPENCLAW_GATEWAY_URL=http://localhost:18789
|
| 46 |
+
```
|
| 47 |
+
|
| 48 |
+
## Usage
|
| 49 |
+
|
| 50 |
+
### Console Mode
|
| 51 |
+
```bash
|
| 52 |
+
reachy-openclaw
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
### Web UI Mode
|
| 56 |
+
```bash
|
| 57 |
+
reachy-openclaw --gradio
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
### As Reachy Mini App
|
| 61 |
+
Install from the robot's dashboard app store.
|
| 62 |
+
|
| 63 |
+
## Features
|
| 64 |
+
|
| 65 |
+
### Voice Conversation
|
| 66 |
+
Real-time voice interaction using OpenAI's Realtime API for natural, low-latency conversation.
|
| 67 |
+
|
| 68 |
+
### Expressive Movements
|
| 69 |
+
The robot automatically:
|
| 70 |
+
- Nods and moves while speaking (audio-driven wobble)
|
| 71 |
+
- Looks toward speakers
|
| 72 |
+
- Expresses emotions through head movements
|
| 73 |
+
- Performs dances when appropriate
|
| 74 |
+
|
| 75 |
+
### Vision
|
| 76 |
+
Ask the robot to describe what it sees:
|
| 77 |
+
> "What do you see in front of you?"
|
| 78 |
+
|
| 79 |
+
### OpenClaw Integration
|
| 80 |
+
Full access to OpenClaw's tool ecosystem:
|
| 81 |
+
> "What's the weather like today?"
|
| 82 |
+
> "Add a reminder for tomorrow"
|
| 83 |
+
> "Turn on the living room lights"
|
| 84 |
+
|
| 85 |
+
## Links
|
| 86 |
+
|
| 87 |
+
- [GitHub Repository](https://github.com/yourusername/reachy_mini_openclaw)
|
| 88 |
+
- [Reachy Mini SDK](https://github.com/pollen-robotics/reachy_mini)
|
| 89 |
+
- [OpenClaw Documentation](https://docs.openclaw.ai)
|
| 90 |
+
|
| 91 |
+
## Author
|
| 92 |
+
|
| 93 |
+
Tom
|
| 94 |
+
|
| 95 |
+
## License
|
| 96 |
+
|
| 97 |
+
Apache 2.0
|
pyproject.toml
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=61.0", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "clawbody"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "ClawBody - Give your OpenClaw AI agent a physical robot body with Reachy Mini. Voice conversation powered by OpenAI Realtime API with expressive movements."
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
license = {text = "Apache-2.0"}
|
| 11 |
+
requires-python = ">=3.11"
|
| 12 |
+
authors = [
|
| 13 |
+
{name = "Tom", email = "tom@example.com"}
|
| 14 |
+
]
|
| 15 |
+
keywords = [
|
| 16 |
+
"clawbody",
|
| 17 |
+
"reachy-mini",
|
| 18 |
+
"openclaw",
|
| 19 |
+
"clawson",
|
| 20 |
+
"robotics",
|
| 21 |
+
"ai-assistant",
|
| 22 |
+
"openai-realtime",
|
| 23 |
+
"voice-conversation",
|
| 24 |
+
"expressive-robot",
|
| 25 |
+
"embodied-ai"
|
| 26 |
+
]
|
| 27 |
+
classifiers = [
|
| 28 |
+
"Development Status :: 4 - Beta",
|
| 29 |
+
"Intended Audience :: Developers",
|
| 30 |
+
"License :: OSI Approved :: Apache Software License",
|
| 31 |
+
"Programming Language :: Python :: 3",
|
| 32 |
+
"Programming Language :: Python :: 3.11",
|
| 33 |
+
"Programming Language :: Python :: 3.12",
|
| 34 |
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
| 35 |
+
"Topic :: Scientific/Engineering :: Human Machine Interfaces",
|
| 36 |
+
]
|
| 37 |
+
dependencies = [
|
| 38 |
+
# OpenAI Realtime API
|
| 39 |
+
"openai>=1.50.0",
|
| 40 |
+
|
| 41 |
+
# Audio streaming
|
| 42 |
+
"fastrtc>=0.0.17",
|
| 43 |
+
"numpy",
|
| 44 |
+
"scipy",
|
| 45 |
+
|
| 46 |
+
# OpenClaw gateway client
|
| 47 |
+
"httpx>=0.27.0",
|
| 48 |
+
"httpx-sse>=0.4.0",
|
| 49 |
+
"websockets>=12.0",
|
| 50 |
+
|
| 51 |
+
# Gradio UI
|
| 52 |
+
"gradio>=4.0.0",
|
| 53 |
+
|
| 54 |
+
# Environment
|
| 55 |
+
"python-dotenv",
|
| 56 |
+
]
|
| 57 |
+
|
| 58 |
+
# Note: reachy-mini SDK must be installed separately from the robot or GitHub:
|
| 59 |
+
# pip install git+https://github.com/pollen-robotics/reachy_mini.git
|
| 60 |
+
# Or on the robot, it's pre-installed.
|
| 61 |
+
|
| 62 |
+
[project.optional-dependencies]
|
| 63 |
+
wireless = [
|
| 64 |
+
"pygobject",
|
| 65 |
+
]
|
| 66 |
+
vision = [
|
| 67 |
+
"opencv-python",
|
| 68 |
+
"ultralytics",
|
| 69 |
+
]
|
| 70 |
+
dev = [
|
| 71 |
+
"pytest",
|
| 72 |
+
"pytest-asyncio",
|
| 73 |
+
"ruff",
|
| 74 |
+
"mypy",
|
| 75 |
+
]
|
| 76 |
+
|
| 77 |
+
[project.scripts]
|
| 78 |
+
clawbody = "reachy_mini_openclaw.main:main"
|
| 79 |
+
|
| 80 |
+
[project.entry-points."reachy_mini_apps"]
|
| 81 |
+
clawbody = "reachy_mini_openclaw.main:ClawBodyApp"
|
| 82 |
+
|
| 83 |
+
[project.urls]
|
| 84 |
+
Homepage = "https://github.com/yourusername/clawbody"
|
| 85 |
+
Documentation = "https://github.com/yourusername/clawbody#readme"
|
| 86 |
+
Repository = "https://github.com/yourusername/clawbody"
|
| 87 |
+
Issues = "https://github.com/yourusername/clawbody/issues"
|
| 88 |
+
|
| 89 |
+
[tool.setuptools.packages.find]
|
| 90 |
+
where = ["src"]
|
| 91 |
+
|
| 92 |
+
[tool.ruff]
|
| 93 |
+
line-length = 120
|
| 94 |
+
target-version = "py311"
|
| 95 |
+
|
| 96 |
+
[tool.ruff.lint]
|
| 97 |
+
select = ["E", "F", "I", "N", "W", "UP"]
|
| 98 |
+
ignore = ["E501"]
|
| 99 |
+
|
| 100 |
+
[tool.mypy]
|
| 101 |
+
python_version = "3.11"
|
| 102 |
+
warn_return_any = true
|
| 103 |
+
warn_unused_configs = true
|
| 104 |
+
ignore_missing_imports = true
|
src/reachy_mini_openclaw/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reachy Mini OpenClaw - Give your OpenClaw AI agent a physical presence.
|
| 2 |
+
|
| 3 |
+
This package combines OpenAI's Realtime API for responsive voice conversation
|
| 4 |
+
with Reachy Mini's expressive robot movements, allowing your OpenClaw agent
|
| 5 |
+
to see, hear, and speak through a physical robot body.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
__version__ = "0.1.0"
|
| 9 |
+
__author__ = "Tom"
|
src/reachy_mini_openclaw/audio/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio processing modules for Reachy Mini OpenClaw."""
|
| 2 |
+
|
| 3 |
+
from reachy_mini_openclaw.audio.head_wobbler import HeadWobbler
|
| 4 |
+
|
| 5 |
+
__all__ = ["HeadWobbler"]
|
src/reachy_mini_openclaw/audio/head_wobbler.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio-driven head movement for natural speech animation.
|
| 2 |
+
|
| 3 |
+
This module analyzes audio output in real-time and generates subtle head
|
| 4 |
+
movements that make the robot appear more expressive and alive while speaking.
|
| 5 |
+
|
| 6 |
+
The wobble is generated based on:
|
| 7 |
+
- Audio amplitude (volume) -> vertical movement
|
| 8 |
+
- Frequency content -> horizontal sway
|
| 9 |
+
- Speech rhythm -> timing of movements
|
| 10 |
+
|
| 11 |
+
Design:
|
| 12 |
+
- Runs in a separate thread to avoid blocking the main audio pipeline
|
| 13 |
+
- Uses a circular buffer for smooth interpolation
|
| 14 |
+
- Generates offsets that are added to the primary pose by MovementManager
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import base64
|
| 18 |
+
import logging
|
| 19 |
+
import threading
|
| 20 |
+
import time
|
| 21 |
+
from collections import deque
|
| 22 |
+
from typing import Callable, Optional, Tuple
|
| 23 |
+
|
| 24 |
+
import numpy as np
|
| 25 |
+
from numpy.typing import NDArray
|
| 26 |
+
|
| 27 |
+
logger = logging.getLogger(__name__)
|
| 28 |
+
|
| 29 |
+
# Type alias for speech offsets: (x, y, z, roll, pitch, yaw)
|
| 30 |
+
SpeechOffsets = Tuple[float, float, float, float, float, float]
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class HeadWobbler:
|
| 34 |
+
"""Generate audio-driven head movements for expressive speech.
|
| 35 |
+
|
| 36 |
+
The wobbler analyzes incoming audio and produces subtle head movements
|
| 37 |
+
that are synchronized with speech patterns, making the robot appear
|
| 38 |
+
more natural and engaged during conversation.
|
| 39 |
+
|
| 40 |
+
Example:
|
| 41 |
+
def apply_offsets(offsets):
|
| 42 |
+
movement_manager.set_speech_offsets(offsets)
|
| 43 |
+
|
| 44 |
+
wobbler = HeadWobbler(set_speech_offsets=apply_offsets)
|
| 45 |
+
wobbler.start()
|
| 46 |
+
|
| 47 |
+
# Feed audio as it's played
|
| 48 |
+
wobbler.feed(base64_audio_chunk)
|
| 49 |
+
|
| 50 |
+
wobbler.stop()
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
def __init__(
|
| 54 |
+
self,
|
| 55 |
+
set_speech_offsets: Callable[[SpeechOffsets], None],
|
| 56 |
+
sample_rate: int = 24000,
|
| 57 |
+
update_rate: float = 30.0, # Hz
|
| 58 |
+
):
|
| 59 |
+
"""Initialize the head wobbler.
|
| 60 |
+
|
| 61 |
+
Args:
|
| 62 |
+
set_speech_offsets: Callback to apply offsets to the movement system
|
| 63 |
+
sample_rate: Expected audio sample rate (Hz)
|
| 64 |
+
update_rate: How often to update offsets (Hz)
|
| 65 |
+
"""
|
| 66 |
+
self.set_speech_offsets = set_speech_offsets
|
| 67 |
+
self.sample_rate = sample_rate
|
| 68 |
+
self.update_period = 1.0 / update_rate
|
| 69 |
+
|
| 70 |
+
# Audio analysis parameters
|
| 71 |
+
self.amplitude_scale = 0.008 # Max displacement in meters
|
| 72 |
+
self.roll_scale = 0.15 # Max roll in radians
|
| 73 |
+
self.pitch_scale = 0.08 # Max pitch in radians
|
| 74 |
+
self.smoothing = 0.3 # Smoothing factor (0-1)
|
| 75 |
+
|
| 76 |
+
# State
|
| 77 |
+
self._audio_buffer: deque[NDArray[np.float32]] = deque(maxlen=10)
|
| 78 |
+
self._buffer_lock = threading.Lock()
|
| 79 |
+
self._current_amplitude = 0.0
|
| 80 |
+
self._current_offsets: SpeechOffsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
|
| 81 |
+
|
| 82 |
+
# Thread control
|
| 83 |
+
self._stop_event = threading.Event()
|
| 84 |
+
self._thread: Optional[threading.Thread] = None
|
| 85 |
+
self._last_feed_time = 0.0
|
| 86 |
+
self._is_speaking = False
|
| 87 |
+
|
| 88 |
+
# Decay parameters for smooth return to neutral
|
| 89 |
+
self._decay_rate = 3.0 # How fast to decay when not speaking
|
| 90 |
+
self._speech_timeout = 0.3 # Seconds of silence before decay starts
|
| 91 |
+
|
| 92 |
+
def start(self) -> None:
|
| 93 |
+
"""Start the wobbler thread."""
|
| 94 |
+
if self._thread is not None and self._thread.is_alive():
|
| 95 |
+
logger.warning("HeadWobbler already running")
|
| 96 |
+
return
|
| 97 |
+
|
| 98 |
+
self._stop_event.clear()
|
| 99 |
+
self._thread = threading.Thread(target=self._run_loop, daemon=True)
|
| 100 |
+
self._thread.start()
|
| 101 |
+
logger.debug("HeadWobbler started")
|
| 102 |
+
|
| 103 |
+
def stop(self) -> None:
|
| 104 |
+
"""Stop the wobbler thread."""
|
| 105 |
+
self._stop_event.set()
|
| 106 |
+
if self._thread is not None:
|
| 107 |
+
self._thread.join(timeout=1.0)
|
| 108 |
+
self._thread = None
|
| 109 |
+
|
| 110 |
+
# Reset to neutral
|
| 111 |
+
self.set_speech_offsets((0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
|
| 112 |
+
logger.debug("HeadWobbler stopped")
|
| 113 |
+
|
| 114 |
+
def reset(self) -> None:
|
| 115 |
+
"""Reset the wobbler state (call when speech ends or is interrupted)."""
|
| 116 |
+
with self._buffer_lock:
|
| 117 |
+
self._audio_buffer.clear()
|
| 118 |
+
self._current_amplitude = 0.0
|
| 119 |
+
self._is_speaking = False
|
| 120 |
+
self.set_speech_offsets((0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
|
| 121 |
+
logger.debug("HeadWobbler reset")
|
| 122 |
+
|
| 123 |
+
def feed(self, audio_b64: str) -> None:
|
| 124 |
+
"""Feed audio data to the wobbler.
|
| 125 |
+
|
| 126 |
+
Args:
|
| 127 |
+
audio_b64: Base64-encoded PCM audio (int16)
|
| 128 |
+
"""
|
| 129 |
+
try:
|
| 130 |
+
audio_bytes = base64.b64decode(audio_b64)
|
| 131 |
+
audio_int16 = np.frombuffer(audio_bytes, dtype=np.int16)
|
| 132 |
+
audio_float = audio_int16.astype(np.float32) / 32768.0
|
| 133 |
+
|
| 134 |
+
with self._buffer_lock:
|
| 135 |
+
self._audio_buffer.append(audio_float)
|
| 136 |
+
|
| 137 |
+
self._last_feed_time = time.monotonic()
|
| 138 |
+
self._is_speaking = True
|
| 139 |
+
|
| 140 |
+
except Exception as e:
|
| 141 |
+
logger.debug("Error feeding audio to wobbler: %s", e)
|
| 142 |
+
|
| 143 |
+
def _compute_amplitude(self) -> float:
|
| 144 |
+
"""Compute current audio amplitude from buffer."""
|
| 145 |
+
with self._buffer_lock:
|
| 146 |
+
if not self._audio_buffer:
|
| 147 |
+
return 0.0
|
| 148 |
+
|
| 149 |
+
# Concatenate recent audio
|
| 150 |
+
audio = np.concatenate(list(self._audio_buffer))
|
| 151 |
+
|
| 152 |
+
# RMS amplitude
|
| 153 |
+
rms = np.sqrt(np.mean(audio ** 2))
|
| 154 |
+
return min(1.0, rms * 3.0) # Scale and clamp
|
| 155 |
+
|
| 156 |
+
def _compute_offsets(self, amplitude: float, t: float) -> SpeechOffsets:
|
| 157 |
+
"""Compute head offsets based on amplitude and time.
|
| 158 |
+
|
| 159 |
+
Args:
|
| 160 |
+
amplitude: Current audio amplitude (0-1)
|
| 161 |
+
t: Current time for oscillation
|
| 162 |
+
|
| 163 |
+
Returns:
|
| 164 |
+
Tuple of (x, y, z, roll, pitch, yaw) offsets
|
| 165 |
+
"""
|
| 166 |
+
if amplitude < 0.01:
|
| 167 |
+
return (0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
|
| 168 |
+
|
| 169 |
+
# Vertical bob based on amplitude
|
| 170 |
+
z_offset = amplitude * self.amplitude_scale * np.sin(t * 8.0)
|
| 171 |
+
|
| 172 |
+
# Subtle roll sway
|
| 173 |
+
roll_offset = amplitude * self.roll_scale * np.sin(t * 3.0)
|
| 174 |
+
|
| 175 |
+
# Pitch variation
|
| 176 |
+
pitch_offset = amplitude * self.pitch_scale * np.sin(t * 5.0 + 0.5)
|
| 177 |
+
|
| 178 |
+
# Small yaw drift
|
| 179 |
+
yaw_offset = amplitude * 0.05 * np.sin(t * 2.0)
|
| 180 |
+
|
| 181 |
+
return (0.0, 0.0, z_offset, roll_offset, pitch_offset, yaw_offset)
|
| 182 |
+
|
| 183 |
+
def _run_loop(self) -> None:
|
| 184 |
+
"""Main wobbler loop."""
|
| 185 |
+
start_time = time.monotonic()
|
| 186 |
+
|
| 187 |
+
while not self._stop_event.is_set():
|
| 188 |
+
loop_start = time.monotonic()
|
| 189 |
+
t = loop_start - start_time
|
| 190 |
+
|
| 191 |
+
# Check if we're still receiving audio
|
| 192 |
+
silence_duration = loop_start - self._last_feed_time
|
| 193 |
+
|
| 194 |
+
if silence_duration > self._speech_timeout:
|
| 195 |
+
# Decay amplitude when not speaking
|
| 196 |
+
self._current_amplitude *= np.exp(-self._decay_rate * self.update_period)
|
| 197 |
+
self._is_speaking = False
|
| 198 |
+
else:
|
| 199 |
+
# Compute new amplitude with smoothing
|
| 200 |
+
raw_amplitude = self._compute_amplitude()
|
| 201 |
+
self._current_amplitude = (
|
| 202 |
+
self.smoothing * raw_amplitude +
|
| 203 |
+
(1 - self.smoothing) * self._current_amplitude
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
# Compute and apply offsets
|
| 207 |
+
offsets = self._compute_offsets(self._current_amplitude, t)
|
| 208 |
+
|
| 209 |
+
# Smooth transition between offsets
|
| 210 |
+
new_offsets = tuple(
|
| 211 |
+
self.smoothing * new + (1 - self.smoothing) * old
|
| 212 |
+
for new, old in zip(offsets, self._current_offsets)
|
| 213 |
+
)
|
| 214 |
+
self._current_offsets = new_offsets
|
| 215 |
+
|
| 216 |
+
# Apply to movement system
|
| 217 |
+
self.set_speech_offsets(new_offsets)
|
| 218 |
+
|
| 219 |
+
# Maintain update rate
|
| 220 |
+
elapsed = time.monotonic() - loop_start
|
| 221 |
+
sleep_time = max(0.0, self.update_period - elapsed)
|
| 222 |
+
if sleep_time > 0:
|
| 223 |
+
time.sleep(sleep_time)
|
src/reachy_mini_openclaw/config.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Configuration management for Reachy Mini OpenClaw.
|
| 2 |
+
|
| 3 |
+
Handles environment variables and configuration settings for the application.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from dataclasses import dataclass, field
|
| 9 |
+
from typing import Optional
|
| 10 |
+
|
| 11 |
+
from dotenv import load_dotenv
|
| 12 |
+
|
| 13 |
+
# Load environment variables from .env file
|
| 14 |
+
_project_root = Path(__file__).parent.parent.parent
|
| 15 |
+
load_dotenv(_project_root / ".env")
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@dataclass
|
| 19 |
+
class Config:
|
| 20 |
+
"""Application configuration loaded from environment variables."""
|
| 21 |
+
|
| 22 |
+
# OpenAI Configuration
|
| 23 |
+
OPENAI_API_KEY: str = field(default_factory=lambda: os.getenv("OPENAI_API_KEY", ""))
|
| 24 |
+
OPENAI_MODEL: str = field(default_factory=lambda: os.getenv("OPENAI_MODEL", "gpt-4o-realtime-preview-2024-12-17"))
|
| 25 |
+
OPENAI_VOICE: str = field(default_factory=lambda: os.getenv("OPENAI_VOICE", "cedar"))
|
| 26 |
+
|
| 27 |
+
# OpenClaw Gateway Configuration
|
| 28 |
+
OPENCLAW_GATEWAY_URL: str = field(default_factory=lambda: os.getenv("OPENCLAW_GATEWAY_URL", "http://localhost:18789"))
|
| 29 |
+
OPENCLAW_TOKEN: Optional[str] = field(default_factory=lambda: os.getenv("OPENCLAW_TOKEN"))
|
| 30 |
+
OPENCLAW_AGENT_ID: str = field(default_factory=lambda: os.getenv("OPENCLAW_AGENT_ID", "main"))
|
| 31 |
+
|
| 32 |
+
# Robot Configuration
|
| 33 |
+
ROBOT_NAME: Optional[str] = field(default_factory=lambda: os.getenv("ROBOT_NAME"))
|
| 34 |
+
|
| 35 |
+
# Feature Flags
|
| 36 |
+
ENABLE_OPENCLAW_TOOLS: bool = field(default_factory=lambda: os.getenv("ENABLE_OPENCLAW_TOOLS", "true").lower() == "true")
|
| 37 |
+
ENABLE_CAMERA: bool = field(default_factory=lambda: os.getenv("ENABLE_CAMERA", "true").lower() == "true")
|
| 38 |
+
ENABLE_FACE_TRACKING: bool = field(default_factory=lambda: os.getenv("ENABLE_FACE_TRACKING", "false").lower() == "true")
|
| 39 |
+
|
| 40 |
+
# Custom Profile (for personality customization)
|
| 41 |
+
CUSTOM_PROFILE: Optional[str] = field(default_factory=lambda: os.getenv("REACHY_MINI_CUSTOM_PROFILE"))
|
| 42 |
+
|
| 43 |
+
def validate(self) -> list[str]:
|
| 44 |
+
"""Validate configuration and return list of errors."""
|
| 45 |
+
errors = []
|
| 46 |
+
if not self.OPENAI_API_KEY:
|
| 47 |
+
errors.append("OPENAI_API_KEY is required")
|
| 48 |
+
return errors
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# Global configuration instance
|
| 52 |
+
config = Config()
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def set_custom_profile(profile: Optional[str]) -> None:
|
| 56 |
+
"""Update the custom profile at runtime."""
|
| 57 |
+
global config
|
| 58 |
+
config.CUSTOM_PROFILE = profile
|
| 59 |
+
os.environ["REACHY_MINI_CUSTOM_PROFILE"] = profile or ""
|
src/reachy_mini_openclaw/gradio_app.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gradio web UI for Reachy Mini OpenClaw.
|
| 2 |
+
|
| 3 |
+
This module provides a web interface for:
|
| 4 |
+
- Viewing conversation transcripts
|
| 5 |
+
- Configuring the assistant personality
|
| 6 |
+
- Monitoring robot status
|
| 7 |
+
- Manual control options
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import os
|
| 11 |
+
import logging
|
| 12 |
+
from typing import Optional
|
| 13 |
+
|
| 14 |
+
import gradio as gr
|
| 15 |
+
|
| 16 |
+
logger = logging.getLogger(__name__)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def launch_gradio(
|
| 20 |
+
gateway_url: str = "http://localhost:18789",
|
| 21 |
+
robot_name: Optional[str] = None,
|
| 22 |
+
enable_camera: bool = True,
|
| 23 |
+
enable_openclaw: bool = True,
|
| 24 |
+
share: bool = False,
|
| 25 |
+
) -> None:
|
| 26 |
+
"""Launch the Gradio web UI.
|
| 27 |
+
|
| 28 |
+
Args:
|
| 29 |
+
gateway_url: OpenClaw gateway URL
|
| 30 |
+
robot_name: Robot name for connection
|
| 31 |
+
enable_camera: Whether to enable camera
|
| 32 |
+
enable_openclaw: Whether to enable OpenClaw
|
| 33 |
+
share: Whether to create a public URL
|
| 34 |
+
"""
|
| 35 |
+
from reachy_mini_openclaw.prompts import get_available_profiles, save_custom_profile
|
| 36 |
+
from reachy_mini_openclaw.config import set_custom_profile, config
|
| 37 |
+
|
| 38 |
+
# State
|
| 39 |
+
app_instance = None
|
| 40 |
+
|
| 41 |
+
def start_conversation():
|
| 42 |
+
"""Start the conversation."""
|
| 43 |
+
nonlocal app_instance
|
| 44 |
+
|
| 45 |
+
from reachy_mini_openclaw.main import ReachyOpenClawCore
|
| 46 |
+
import asyncio
|
| 47 |
+
import threading
|
| 48 |
+
|
| 49 |
+
if app_instance is not None:
|
| 50 |
+
return "Already running"
|
| 51 |
+
|
| 52 |
+
try:
|
| 53 |
+
app_instance = ReachyOpenClawCore(
|
| 54 |
+
gateway_url=gateway_url,
|
| 55 |
+
robot_name=robot_name,
|
| 56 |
+
enable_camera=enable_camera,
|
| 57 |
+
enable_openclaw=enable_openclaw,
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
# Run in background thread
|
| 61 |
+
def run_app():
|
| 62 |
+
loop = asyncio.new_event_loop()
|
| 63 |
+
asyncio.set_event_loop(loop)
|
| 64 |
+
try:
|
| 65 |
+
loop.run_until_complete(app_instance.run())
|
| 66 |
+
except Exception as e:
|
| 67 |
+
logger.error("App error: %s", e)
|
| 68 |
+
finally:
|
| 69 |
+
loop.close()
|
| 70 |
+
|
| 71 |
+
thread = threading.Thread(target=run_app, daemon=True)
|
| 72 |
+
thread.start()
|
| 73 |
+
|
| 74 |
+
return "Started successfully"
|
| 75 |
+
except Exception as e:
|
| 76 |
+
return f"Error: {e}"
|
| 77 |
+
|
| 78 |
+
def stop_conversation():
|
| 79 |
+
"""Stop the conversation."""
|
| 80 |
+
nonlocal app_instance
|
| 81 |
+
|
| 82 |
+
if app_instance is None:
|
| 83 |
+
return "Not running"
|
| 84 |
+
|
| 85 |
+
try:
|
| 86 |
+
app_instance.stop()
|
| 87 |
+
app_instance = None
|
| 88 |
+
return "Stopped"
|
| 89 |
+
except Exception as e:
|
| 90 |
+
return f"Error: {e}"
|
| 91 |
+
|
| 92 |
+
def apply_profile(profile_name):
|
| 93 |
+
"""Apply a personality profile."""
|
| 94 |
+
set_custom_profile(profile_name if profile_name else None)
|
| 95 |
+
return f"Applied profile: {profile_name or 'default'}"
|
| 96 |
+
|
| 97 |
+
def save_profile(name, instructions):
|
| 98 |
+
"""Save a new profile."""
|
| 99 |
+
if save_custom_profile(name, instructions):
|
| 100 |
+
return f"Saved profile: {name}"
|
| 101 |
+
return "Error saving profile"
|
| 102 |
+
|
| 103 |
+
# Build UI
|
| 104 |
+
with gr.Blocks(title="Reachy Mini OpenClaw") as demo:
|
| 105 |
+
gr.Markdown("""
|
| 106 |
+
# 🤖 Reachy Mini OpenClaw
|
| 107 |
+
|
| 108 |
+
Give your OpenClaw AI agent a physical presence with Reachy Mini.
|
| 109 |
+
Using OpenAI Realtime API for responsive voice conversation.
|
| 110 |
+
""")
|
| 111 |
+
|
| 112 |
+
with gr.Tab("Conversation"):
|
| 113 |
+
with gr.Row():
|
| 114 |
+
start_btn = gr.Button("▶️ Start", variant="primary")
|
| 115 |
+
stop_btn = gr.Button("⏹️ Stop", variant="secondary")
|
| 116 |
+
|
| 117 |
+
status_text = gr.Textbox(label="Status", interactive=False)
|
| 118 |
+
|
| 119 |
+
transcript = gr.Chatbot(label="Conversation", height=400)
|
| 120 |
+
|
| 121 |
+
start_btn.click(start_conversation, outputs=[status_text])
|
| 122 |
+
stop_btn.click(stop_conversation, outputs=[status_text])
|
| 123 |
+
|
| 124 |
+
with gr.Tab("Personality"):
|
| 125 |
+
profiles = get_available_profiles()
|
| 126 |
+
profile_dropdown = gr.Dropdown(
|
| 127 |
+
choices=[""] + profiles,
|
| 128 |
+
label="Select Profile",
|
| 129 |
+
value=""
|
| 130 |
+
)
|
| 131 |
+
apply_btn = gr.Button("Apply Profile")
|
| 132 |
+
profile_status = gr.Textbox(label="Status", interactive=False)
|
| 133 |
+
|
| 134 |
+
apply_btn.click(
|
| 135 |
+
apply_profile,
|
| 136 |
+
inputs=[profile_dropdown],
|
| 137 |
+
outputs=[profile_status]
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
gr.Markdown("### Create New Profile")
|
| 141 |
+
new_name = gr.Textbox(label="Profile Name")
|
| 142 |
+
new_instructions = gr.Textbox(
|
| 143 |
+
label="Instructions",
|
| 144 |
+
lines=10,
|
| 145 |
+
placeholder="Enter the system prompt for this personality..."
|
| 146 |
+
)
|
| 147 |
+
save_btn = gr.Button("Save Profile")
|
| 148 |
+
save_status = gr.Textbox(label="Save Status", interactive=False)
|
| 149 |
+
|
| 150 |
+
save_btn.click(
|
| 151 |
+
save_profile,
|
| 152 |
+
inputs=[new_name, new_instructions],
|
| 153 |
+
outputs=[save_status]
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
with gr.Tab("Settings"):
|
| 157 |
+
gr.Markdown(f"""
|
| 158 |
+
### Current Configuration
|
| 159 |
+
|
| 160 |
+
- **OpenClaw Gateway**: {gateway_url}
|
| 161 |
+
- **OpenAI Model**: {config.OPENAI_MODEL}
|
| 162 |
+
- **Voice**: {config.OPENAI_VOICE}
|
| 163 |
+
- **Camera Enabled**: {enable_camera}
|
| 164 |
+
- **OpenClaw Enabled**: {enable_openclaw}
|
| 165 |
+
|
| 166 |
+
Edit `.env` file to change these settings.
|
| 167 |
+
""")
|
| 168 |
+
|
| 169 |
+
with gr.Tab("About"):
|
| 170 |
+
gr.Markdown("""
|
| 171 |
+
## About Reachy Mini OpenClaw
|
| 172 |
+
|
| 173 |
+
This application combines:
|
| 174 |
+
|
| 175 |
+
- **OpenAI Realtime API** for ultra-low-latency voice conversation
|
| 176 |
+
- **OpenClaw Gateway** for extended AI capabilities (web, calendar, smart home, etc.)
|
| 177 |
+
- **Reachy Mini Robot** for physical embodiment with expressive movements
|
| 178 |
+
|
| 179 |
+
### Features
|
| 180 |
+
|
| 181 |
+
- 🎤 Real-time voice conversation
|
| 182 |
+
- 👀 Camera-based vision
|
| 183 |
+
- 💃 Expressive robot movements
|
| 184 |
+
- 🔧 Tool integration via OpenClaw
|
| 185 |
+
- 🎭 Customizable personalities
|
| 186 |
+
|
| 187 |
+
### Links
|
| 188 |
+
|
| 189 |
+
- [Reachy Mini SDK](https://github.com/pollen-robotics/reachy_mini)
|
| 190 |
+
- [OpenClaw](https://github.com/openclaw/openclaw)
|
| 191 |
+
- [OpenAI Realtime API](https://platform.openai.com/docs/guides/realtime)
|
| 192 |
+
""")
|
| 193 |
+
|
| 194 |
+
demo.launch(share=share, server_name="0.0.0.0", server_port=7860)
|
src/reachy_mini_openclaw/main.py
ADDED
|
@@ -0,0 +1,420 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ClawBody - Give your OpenClaw AI agent a physical robot body.
|
| 2 |
+
|
| 3 |
+
This module provides the main application that connects:
|
| 4 |
+
- OpenAI Realtime API for voice I/O (speech recognition + TTS)
|
| 5 |
+
- OpenClaw Gateway for AI intelligence (Clawson's brain)
|
| 6 |
+
- Reachy Mini robot for physical embodiment
|
| 7 |
+
|
| 8 |
+
Usage:
|
| 9 |
+
# Console mode (direct audio)
|
| 10 |
+
clawbody
|
| 11 |
+
|
| 12 |
+
# With Gradio UI
|
| 13 |
+
clawbody --gradio
|
| 14 |
+
|
| 15 |
+
# With debug logging
|
| 16 |
+
clawbody --debug
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import os
|
| 20 |
+
import sys
|
| 21 |
+
import time
|
| 22 |
+
import asyncio
|
| 23 |
+
import logging
|
| 24 |
+
import argparse
|
| 25 |
+
import threading
|
| 26 |
+
from pathlib import Path
|
| 27 |
+
from typing import Optional
|
| 28 |
+
|
| 29 |
+
from dotenv import load_dotenv
|
| 30 |
+
|
| 31 |
+
# Load environment from project root (override=True ensures .env takes precedence)
|
| 32 |
+
_project_root = Path(__file__).parent.parent.parent
|
| 33 |
+
load_dotenv(_project_root / ".env", override=True)
|
| 34 |
+
|
| 35 |
+
logger = logging.getLogger(__name__)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def setup_logging(debug: bool = False) -> None:
|
| 39 |
+
"""Configure logging for the application.
|
| 40 |
+
|
| 41 |
+
Args:
|
| 42 |
+
debug: Enable debug level logging
|
| 43 |
+
"""
|
| 44 |
+
level = logging.DEBUG if debug else logging.INFO
|
| 45 |
+
logging.basicConfig(
|
| 46 |
+
level=level,
|
| 47 |
+
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
| 48 |
+
datefmt="%H:%M:%S",
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
# Reduce noise from libraries
|
| 52 |
+
if not debug:
|
| 53 |
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
| 54 |
+
logging.getLogger("websockets").setLevel(logging.WARNING)
|
| 55 |
+
logging.getLogger("openai").setLevel(logging.WARNING)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def parse_args() -> argparse.Namespace:
|
| 59 |
+
"""Parse command line arguments.
|
| 60 |
+
|
| 61 |
+
Returns:
|
| 62 |
+
Parsed arguments namespace
|
| 63 |
+
"""
|
| 64 |
+
parser = argparse.ArgumentParser(
|
| 65 |
+
description="ClawBody - Give your OpenClaw AI agent a physical robot body",
|
| 66 |
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
| 67 |
+
epilog="""
|
| 68 |
+
Examples:
|
| 69 |
+
# Run in console mode
|
| 70 |
+
clawbody
|
| 71 |
+
|
| 72 |
+
# Run with Gradio web UI
|
| 73 |
+
clawbody --gradio
|
| 74 |
+
|
| 75 |
+
# Connect to specific robot
|
| 76 |
+
clawbody --robot-name my-reachy
|
| 77 |
+
|
| 78 |
+
# Use different OpenClaw gateway
|
| 79 |
+
clawbody --gateway-url http://192.168.1.100:18790
|
| 80 |
+
"""
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
parser.add_argument(
|
| 84 |
+
"--debug",
|
| 85 |
+
action="store_true",
|
| 86 |
+
help="Enable debug logging"
|
| 87 |
+
)
|
| 88 |
+
parser.add_argument(
|
| 89 |
+
"--gradio",
|
| 90 |
+
action="store_true",
|
| 91 |
+
help="Launch Gradio web UI instead of console mode"
|
| 92 |
+
)
|
| 93 |
+
parser.add_argument(
|
| 94 |
+
"--robot-name",
|
| 95 |
+
type=str,
|
| 96 |
+
help="Robot name for connection (default: auto-discover)"
|
| 97 |
+
)
|
| 98 |
+
parser.add_argument(
|
| 99 |
+
"--gateway-url",
|
| 100 |
+
type=str,
|
| 101 |
+
default=os.getenv("OPENCLAW_GATEWAY_URL", "http://localhost:18789"),
|
| 102 |
+
help="OpenClaw gateway URL (from OPENCLAW_GATEWAY_URL env or default)"
|
| 103 |
+
)
|
| 104 |
+
parser.add_argument(
|
| 105 |
+
"--no-camera",
|
| 106 |
+
action="store_true",
|
| 107 |
+
help="Disable camera functionality"
|
| 108 |
+
)
|
| 109 |
+
parser.add_argument(
|
| 110 |
+
"--no-openclaw",
|
| 111 |
+
action="store_true",
|
| 112 |
+
help="Disable OpenClaw integration"
|
| 113 |
+
)
|
| 114 |
+
parser.add_argument(
|
| 115 |
+
"--profile",
|
| 116 |
+
type=str,
|
| 117 |
+
help="Custom personality profile to use"
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
return parser.parse_args()
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class ClawBodyCore:
|
| 124 |
+
"""ClawBody core application controller.
|
| 125 |
+
|
| 126 |
+
This class orchestrates all components:
|
| 127 |
+
- Reachy Mini robot connection and movement control
|
| 128 |
+
- OpenAI Realtime API for voice I/O
|
| 129 |
+
- OpenClaw gateway bridge for AI intelligence
|
| 130 |
+
- Audio input/output loops
|
| 131 |
+
"""
|
| 132 |
+
|
| 133 |
+
def __init__(
|
| 134 |
+
self,
|
| 135 |
+
gateway_url: str = "http://localhost:18789",
|
| 136 |
+
robot_name: Optional[str] = None,
|
| 137 |
+
enable_camera: bool = True,
|
| 138 |
+
enable_openclaw: bool = True,
|
| 139 |
+
robot: Optional["ReachyMini"] = None,
|
| 140 |
+
external_stop_event: Optional[threading.Event] = None,
|
| 141 |
+
):
|
| 142 |
+
"""Initialize the application.
|
| 143 |
+
|
| 144 |
+
Args:
|
| 145 |
+
gateway_url: OpenClaw gateway URL
|
| 146 |
+
robot_name: Optional robot name for connection
|
| 147 |
+
enable_camera: Whether to enable camera functionality
|
| 148 |
+
enable_openclaw: Whether to enable OpenClaw integration
|
| 149 |
+
robot: Optional pre-initialized robot (for app framework)
|
| 150 |
+
external_stop_event: Optional external stop event
|
| 151 |
+
"""
|
| 152 |
+
from reachy_mini import ReachyMini
|
| 153 |
+
from reachy_mini_openclaw.config import config
|
| 154 |
+
from reachy_mini_openclaw.moves import MovementManager
|
| 155 |
+
from reachy_mini_openclaw.audio.head_wobbler import HeadWobbler
|
| 156 |
+
from reachy_mini_openclaw.openclaw_bridge import OpenClawBridge
|
| 157 |
+
from reachy_mini_openclaw.tools.core_tools import ToolDependencies
|
| 158 |
+
from reachy_mini_openclaw.openai_realtime import OpenAIRealtimeHandler
|
| 159 |
+
|
| 160 |
+
self.gateway_url = gateway_url
|
| 161 |
+
self._external_stop_event = external_stop_event
|
| 162 |
+
self._owns_robot = robot is None
|
| 163 |
+
|
| 164 |
+
# Validate configuration
|
| 165 |
+
errors = config.validate()
|
| 166 |
+
if errors:
|
| 167 |
+
for error in errors:
|
| 168 |
+
logger.error("Config error: %s", error)
|
| 169 |
+
sys.exit(1)
|
| 170 |
+
|
| 171 |
+
# Connect to robot
|
| 172 |
+
if robot is not None:
|
| 173 |
+
self.robot = robot
|
| 174 |
+
logger.info("Using provided Reachy Mini instance")
|
| 175 |
+
else:
|
| 176 |
+
logger.info("Connecting to Reachy Mini...")
|
| 177 |
+
robot_kwargs = {}
|
| 178 |
+
if robot_name:
|
| 179 |
+
robot_kwargs["robot_name"] = robot_name
|
| 180 |
+
|
| 181 |
+
try:
|
| 182 |
+
self.robot = ReachyMini(**robot_kwargs)
|
| 183 |
+
except TimeoutError as e:
|
| 184 |
+
logger.error("Connection timeout: %s", e)
|
| 185 |
+
logger.error("Check that the robot is powered on and reachable.")
|
| 186 |
+
sys.exit(1)
|
| 187 |
+
except Exception as e:
|
| 188 |
+
logger.error("Robot connection failed: %s", e)
|
| 189 |
+
sys.exit(1)
|
| 190 |
+
|
| 191 |
+
logger.info("Connected to robot: %s", self.robot.client.get_status())
|
| 192 |
+
|
| 193 |
+
# Initialize movement system
|
| 194 |
+
logger.info("Initializing movement system...")
|
| 195 |
+
self.movement_manager = MovementManager(current_robot=self.robot)
|
| 196 |
+
self.head_wobbler = HeadWobbler(
|
| 197 |
+
set_speech_offsets=self.movement_manager.set_speech_offsets
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
# Initialize OpenClaw bridge
|
| 201 |
+
self.openclaw_bridge = None
|
| 202 |
+
if enable_openclaw:
|
| 203 |
+
logger.info("Initializing OpenClaw bridge...")
|
| 204 |
+
self.openclaw_bridge = OpenClawBridge(
|
| 205 |
+
gateway_url=gateway_url,
|
| 206 |
+
gateway_token=config.OPENCLAW_TOKEN,
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
# Camera worker (optional)
|
| 210 |
+
self.camera_worker = None
|
| 211 |
+
if enable_camera:
|
| 212 |
+
# Camera worker would be initialized here if needed
|
| 213 |
+
# For now, we use the robot's built-in camera access
|
| 214 |
+
pass
|
| 215 |
+
|
| 216 |
+
# Create tool dependencies
|
| 217 |
+
self.deps = ToolDependencies(
|
| 218 |
+
movement_manager=self.movement_manager,
|
| 219 |
+
head_wobbler=self.head_wobbler,
|
| 220 |
+
robot=self.robot,
|
| 221 |
+
camera_worker=self.camera_worker,
|
| 222 |
+
openclaw_bridge=self.openclaw_bridge,
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
# Initialize OpenAI Realtime handler with OpenClaw bridge
|
| 226 |
+
self.handler = OpenAIRealtimeHandler(
|
| 227 |
+
deps=self.deps,
|
| 228 |
+
openclaw_bridge=self.openclaw_bridge,
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
# State
|
| 232 |
+
self._stop_event = asyncio.Event()
|
| 233 |
+
self._tasks: list[asyncio.Task] = []
|
| 234 |
+
|
| 235 |
+
def _should_stop(self) -> bool:
|
| 236 |
+
"""Check if we should stop."""
|
| 237 |
+
if self._stop_event.is_set():
|
| 238 |
+
return True
|
| 239 |
+
if self._external_stop_event is not None and self._external_stop_event.is_set():
|
| 240 |
+
return True
|
| 241 |
+
return False
|
| 242 |
+
|
| 243 |
+
async def record_loop(self) -> None:
|
| 244 |
+
"""Read audio from robot microphone and send to handler."""
|
| 245 |
+
input_sr = self.robot.media.get_input_audio_samplerate()
|
| 246 |
+
logger.info("Recording at %d Hz", input_sr)
|
| 247 |
+
|
| 248 |
+
while not self._should_stop():
|
| 249 |
+
audio_frame = self.robot.media.get_audio_sample()
|
| 250 |
+
if audio_frame is not None:
|
| 251 |
+
await self.handler.receive((input_sr, audio_frame))
|
| 252 |
+
await asyncio.sleep(0.01)
|
| 253 |
+
|
| 254 |
+
async def play_loop(self) -> None:
|
| 255 |
+
"""Play audio from handler through robot speakers."""
|
| 256 |
+
output_sr = self.robot.media.get_output_audio_samplerate()
|
| 257 |
+
logger.info("Playing at %d Hz", output_sr)
|
| 258 |
+
|
| 259 |
+
while not self._should_stop():
|
| 260 |
+
output = await self.handler.emit()
|
| 261 |
+
if output is not None:
|
| 262 |
+
if isinstance(output, tuple):
|
| 263 |
+
input_sr, audio_data = output
|
| 264 |
+
|
| 265 |
+
# Convert to float32 and normalize (OpenAI sends int16)
|
| 266 |
+
audio_data = audio_data.flatten().astype("float32") / 32768.0
|
| 267 |
+
|
| 268 |
+
# Reduce volume to prevent distortion (0.5 = 50% volume)
|
| 269 |
+
audio_data = audio_data * 0.5
|
| 270 |
+
|
| 271 |
+
# Resample if needed
|
| 272 |
+
if input_sr != output_sr:
|
| 273 |
+
from scipy.signal import resample
|
| 274 |
+
num_samples = int(len(audio_data) * output_sr / input_sr)
|
| 275 |
+
audio_data = resample(audio_data, num_samples).astype("float32")
|
| 276 |
+
|
| 277 |
+
self.robot.media.push_audio_sample(audio_data)
|
| 278 |
+
# else: it's an AdditionalOutputs (transcript) - handle in UI mode
|
| 279 |
+
|
| 280 |
+
await asyncio.sleep(0.01)
|
| 281 |
+
|
| 282 |
+
async def run(self) -> None:
|
| 283 |
+
"""Run the main application loop."""
|
| 284 |
+
# Test OpenClaw connection
|
| 285 |
+
if self.openclaw_bridge is not None:
|
| 286 |
+
connected = await self.openclaw_bridge.connect()
|
| 287 |
+
if connected:
|
| 288 |
+
logger.info("OpenClaw gateway connected")
|
| 289 |
+
else:
|
| 290 |
+
logger.warning("OpenClaw gateway not available - some features disabled")
|
| 291 |
+
|
| 292 |
+
# Start movement system
|
| 293 |
+
logger.info("Starting movement system...")
|
| 294 |
+
self.movement_manager.start()
|
| 295 |
+
self.head_wobbler.start()
|
| 296 |
+
|
| 297 |
+
# Start audio
|
| 298 |
+
logger.info("Starting audio...")
|
| 299 |
+
self.robot.media.start_recording()
|
| 300 |
+
self.robot.media.start_playing()
|
| 301 |
+
time.sleep(1) # Let pipelines initialize
|
| 302 |
+
|
| 303 |
+
logger.info("Ready! Speak to me...")
|
| 304 |
+
|
| 305 |
+
# Start OpenAI handler in background
|
| 306 |
+
handler_task = asyncio.create_task(self.handler.start_up(), name="openai-handler")
|
| 307 |
+
|
| 308 |
+
# Start audio loops
|
| 309 |
+
self._tasks = [
|
| 310 |
+
handler_task,
|
| 311 |
+
asyncio.create_task(self.record_loop(), name="record-loop"),
|
| 312 |
+
asyncio.create_task(self.play_loop(), name="play-loop"),
|
| 313 |
+
]
|
| 314 |
+
|
| 315 |
+
try:
|
| 316 |
+
await asyncio.gather(*self._tasks)
|
| 317 |
+
except asyncio.CancelledError:
|
| 318 |
+
logger.info("Tasks cancelled")
|
| 319 |
+
|
| 320 |
+
def stop(self) -> None:
|
| 321 |
+
"""Stop everything."""
|
| 322 |
+
logger.info("Stopping...")
|
| 323 |
+
self._stop_event.set()
|
| 324 |
+
|
| 325 |
+
# Cancel tasks
|
| 326 |
+
for task in self._tasks:
|
| 327 |
+
if not task.done():
|
| 328 |
+
task.cancel()
|
| 329 |
+
|
| 330 |
+
# Stop movement system
|
| 331 |
+
self.head_wobbler.stop()
|
| 332 |
+
self.movement_manager.stop()
|
| 333 |
+
|
| 334 |
+
# Close resources if we own them
|
| 335 |
+
if self._owns_robot:
|
| 336 |
+
try:
|
| 337 |
+
self.robot.media.close()
|
| 338 |
+
except Exception as e:
|
| 339 |
+
logger.debug("Media close: %s", e)
|
| 340 |
+
self.robot.client.disconnect()
|
| 341 |
+
|
| 342 |
+
logger.info("Stopped")
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
class ClawBodyApp:
|
| 346 |
+
"""ClawBody - Reachy Mini Apps entry point.
|
| 347 |
+
|
| 348 |
+
This class allows ClawBody to be installed and run from
|
| 349 |
+
the Reachy Mini dashboard as a Reachy Mini App.
|
| 350 |
+
"""
|
| 351 |
+
|
| 352 |
+
# No custom settings UI
|
| 353 |
+
custom_app_url: Optional[str] = None
|
| 354 |
+
|
| 355 |
+
def run(self, reachy_mini, stop_event: threading.Event) -> None:
|
| 356 |
+
"""Run ClawBody as a Reachy Mini App.
|
| 357 |
+
|
| 358 |
+
Args:
|
| 359 |
+
reachy_mini: Pre-initialized ReachyMini instance
|
| 360 |
+
stop_event: Threading event to signal stop
|
| 361 |
+
"""
|
| 362 |
+
loop = asyncio.new_event_loop()
|
| 363 |
+
asyncio.set_event_loop(loop)
|
| 364 |
+
|
| 365 |
+
gateway_url = os.getenv("OPENCLAW_GATEWAY_URL", "http://localhost:18789")
|
| 366 |
+
|
| 367 |
+
app = ClawBodyCore(
|
| 368 |
+
gateway_url=gateway_url,
|
| 369 |
+
robot=reachy_mini,
|
| 370 |
+
external_stop_event=stop_event,
|
| 371 |
+
)
|
| 372 |
+
|
| 373 |
+
try:
|
| 374 |
+
loop.run_until_complete(app.run())
|
| 375 |
+
except Exception as e:
|
| 376 |
+
logger.error("Error running app: %s", e)
|
| 377 |
+
finally:
|
| 378 |
+
app.stop()
|
| 379 |
+
loop.close()
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def main() -> None:
|
| 383 |
+
"""Main entry point."""
|
| 384 |
+
args = parse_args()
|
| 385 |
+
setup_logging(args.debug)
|
| 386 |
+
|
| 387 |
+
# Set custom profile if specified
|
| 388 |
+
if args.profile:
|
| 389 |
+
from reachy_mini_openclaw.config import set_custom_profile
|
| 390 |
+
set_custom_profile(args.profile)
|
| 391 |
+
|
| 392 |
+
if args.gradio:
|
| 393 |
+
# Launch Gradio UI
|
| 394 |
+
logger.info("Starting Gradio UI...")
|
| 395 |
+
from reachy_mini_openclaw.gradio_app import launch_gradio
|
| 396 |
+
launch_gradio(
|
| 397 |
+
gateway_url=args.gateway_url,
|
| 398 |
+
robot_name=args.robot_name,
|
| 399 |
+
enable_camera=not args.no_camera,
|
| 400 |
+
enable_openclaw=not args.no_openclaw,
|
| 401 |
+
)
|
| 402 |
+
else:
|
| 403 |
+
# Console mode
|
| 404 |
+
app = ClawBodyCore(
|
| 405 |
+
gateway_url=args.gateway_url,
|
| 406 |
+
robot_name=args.robot_name,
|
| 407 |
+
enable_camera=not args.no_camera,
|
| 408 |
+
enable_openclaw=not args.no_openclaw,
|
| 409 |
+
)
|
| 410 |
+
|
| 411 |
+
try:
|
| 412 |
+
asyncio.run(app.run())
|
| 413 |
+
except KeyboardInterrupt:
|
| 414 |
+
logger.info("Interrupted")
|
| 415 |
+
finally:
|
| 416 |
+
app.stop()
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
if __name__ == "__main__":
|
| 420 |
+
main()
|
src/reachy_mini_openclaw/moves.py
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Movement system for expressive robot control.
|
| 2 |
+
|
| 3 |
+
This module provides a 100Hz control loop for managing robot movements,
|
| 4 |
+
combining sequential primary moves (dances, emotions, head movements) with
|
| 5 |
+
additive secondary moves (speech wobble, face tracking).
|
| 6 |
+
|
| 7 |
+
Architecture:
|
| 8 |
+
- Primary moves are queued and executed sequentially
|
| 9 |
+
- Secondary moves are additive offsets applied on top
|
| 10 |
+
- Single control point via set_target at 100Hz
|
| 11 |
+
- Automatic breathing animation when idle
|
| 12 |
+
|
| 13 |
+
Based on the movement systems from:
|
| 14 |
+
- pollen-robotics/reachy_mini_conversation_app
|
| 15 |
+
- eoai-dev/moltbot_body
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import logging
|
| 21 |
+
import threading
|
| 22 |
+
import time
|
| 23 |
+
from collections import deque
|
| 24 |
+
from dataclasses import dataclass
|
| 25 |
+
from queue import Empty, Queue
|
| 26 |
+
from typing import Any, Dict, Optional, Tuple
|
| 27 |
+
|
| 28 |
+
import numpy as np
|
| 29 |
+
from numpy.typing import NDArray
|
| 30 |
+
from reachy_mini import ReachyMini
|
| 31 |
+
from reachy_mini.motion.move import Move
|
| 32 |
+
from reachy_mini.utils import create_head_pose
|
| 33 |
+
from reachy_mini.utils.interpolation import compose_world_offset, linear_pose_interpolation
|
| 34 |
+
|
| 35 |
+
logger = logging.getLogger(__name__)
|
| 36 |
+
|
| 37 |
+
# Configuration
|
| 38 |
+
CONTROL_LOOP_FREQUENCY_HZ = 100.0
|
| 39 |
+
|
| 40 |
+
# Type definitions
|
| 41 |
+
FullBodyPose = Tuple[NDArray[np.float32], Tuple[float, float], float]
|
| 42 |
+
SpeechOffsets = Tuple[float, float, float, float, float, float]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class BreathingMove(Move):
|
| 46 |
+
"""Continuous breathing animation for idle state."""
|
| 47 |
+
|
| 48 |
+
def __init__(
|
| 49 |
+
self,
|
| 50 |
+
interpolation_start_pose: NDArray[np.float32],
|
| 51 |
+
interpolation_start_antennas: Tuple[float, float],
|
| 52 |
+
interpolation_duration: float = 1.0,
|
| 53 |
+
):
|
| 54 |
+
"""Initialize breathing move.
|
| 55 |
+
|
| 56 |
+
Args:
|
| 57 |
+
interpolation_start_pose: Current head pose to interpolate from
|
| 58 |
+
interpolation_start_antennas: Current antenna positions
|
| 59 |
+
interpolation_duration: Time to blend to neutral (seconds)
|
| 60 |
+
"""
|
| 61 |
+
self.interpolation_start_pose = interpolation_start_pose
|
| 62 |
+
self.interpolation_start_antennas = np.array(interpolation_start_antennas)
|
| 63 |
+
self.interpolation_duration = interpolation_duration
|
| 64 |
+
|
| 65 |
+
# Target neutral pose
|
| 66 |
+
self.neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
|
| 67 |
+
self.neutral_antennas = np.array([0.0, 0.0])
|
| 68 |
+
|
| 69 |
+
# Breathing parameters
|
| 70 |
+
self.breathing_z_amplitude = 0.005 # 5mm gentle movement
|
| 71 |
+
self.breathing_frequency = 0.1 # Hz
|
| 72 |
+
self.antenna_sway_amplitude = np.deg2rad(15) # degrees
|
| 73 |
+
self.antenna_frequency = 0.5 # Hz
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def duration(self) -> float:
|
| 77 |
+
"""Duration of the move (infinite for breathing)."""
|
| 78 |
+
return float("inf")
|
| 79 |
+
|
| 80 |
+
def evaluate(self, t: float) -> tuple:
|
| 81 |
+
"""Evaluate the breathing pose at time t."""
|
| 82 |
+
if t < self.interpolation_duration:
|
| 83 |
+
# Interpolate to neutral
|
| 84 |
+
alpha = t / self.interpolation_duration
|
| 85 |
+
head_pose = linear_pose_interpolation(
|
| 86 |
+
self.interpolation_start_pose,
|
| 87 |
+
self.neutral_head_pose,
|
| 88 |
+
alpha
|
| 89 |
+
)
|
| 90 |
+
antennas = (1 - alpha) * self.interpolation_start_antennas + alpha * self.neutral_antennas
|
| 91 |
+
antennas = antennas.astype(np.float64)
|
| 92 |
+
else:
|
| 93 |
+
# Breathing pattern
|
| 94 |
+
breathing_t = t - self.interpolation_duration
|
| 95 |
+
|
| 96 |
+
z_offset = self.breathing_z_amplitude * np.sin(
|
| 97 |
+
2 * np.pi * self.breathing_frequency * breathing_t
|
| 98 |
+
)
|
| 99 |
+
head_pose = create_head_pose(
|
| 100 |
+
x=0, y=0, z=z_offset,
|
| 101 |
+
roll=0, pitch=0, yaw=0,
|
| 102 |
+
degrees=True, mm=False
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
antenna_sway = self.antenna_sway_amplitude * np.sin(
|
| 106 |
+
2 * np.pi * self.antenna_frequency * breathing_t
|
| 107 |
+
)
|
| 108 |
+
antennas = np.array([antenna_sway, -antenna_sway], dtype=np.float64)
|
| 109 |
+
|
| 110 |
+
return (head_pose, antennas, 0.0)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
class HeadLookMove(Move):
|
| 114 |
+
"""Move to look in a specific direction."""
|
| 115 |
+
|
| 116 |
+
DIRECTIONS = {
|
| 117 |
+
"left": (0, 0, 0, 0, 0, 30), # yaw left
|
| 118 |
+
"right": (0, 0, 0, 0, 0, -30), # yaw right
|
| 119 |
+
"up": (0, 0, 10, 0, 15, 0), # pitch up, z up
|
| 120 |
+
"down": (0, 0, -5, 0, -15, 0), # pitch down, z down
|
| 121 |
+
"front": (0, 0, 0, 0, 0, 0), # neutral
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
def __init__(
|
| 125 |
+
self,
|
| 126 |
+
direction: str,
|
| 127 |
+
start_pose: NDArray[np.float32],
|
| 128 |
+
start_antennas: Tuple[float, float],
|
| 129 |
+
duration: float = 1.0,
|
| 130 |
+
):
|
| 131 |
+
"""Initialize head look move.
|
| 132 |
+
|
| 133 |
+
Args:
|
| 134 |
+
direction: One of 'left', 'right', 'up', 'down', 'front'
|
| 135 |
+
start_pose: Current head pose
|
| 136 |
+
start_antennas: Current antenna positions
|
| 137 |
+
duration: Move duration in seconds
|
| 138 |
+
"""
|
| 139 |
+
self.direction = direction
|
| 140 |
+
self.start_pose = start_pose
|
| 141 |
+
self.start_antennas = np.array(start_antennas)
|
| 142 |
+
self._duration = duration
|
| 143 |
+
|
| 144 |
+
# Get target pose from direction
|
| 145 |
+
params = self.DIRECTIONS.get(direction, self.DIRECTIONS["front"])
|
| 146 |
+
self.target_pose = create_head_pose(
|
| 147 |
+
x=params[0], y=params[1], z=params[2],
|
| 148 |
+
roll=params[3], pitch=params[4], yaw=params[5],
|
| 149 |
+
degrees=True, mm=True
|
| 150 |
+
)
|
| 151 |
+
self.target_antennas = np.array([0.0, 0.0])
|
| 152 |
+
|
| 153 |
+
@property
|
| 154 |
+
def duration(self) -> float:
|
| 155 |
+
return self._duration
|
| 156 |
+
|
| 157 |
+
def evaluate(self, t: float) -> tuple:
|
| 158 |
+
"""Evaluate pose at time t."""
|
| 159 |
+
alpha = min(1.0, t / self._duration)
|
| 160 |
+
# Smooth easing
|
| 161 |
+
alpha = alpha * alpha * (3 - 2 * alpha)
|
| 162 |
+
|
| 163 |
+
head_pose = linear_pose_interpolation(
|
| 164 |
+
self.start_pose,
|
| 165 |
+
self.target_pose,
|
| 166 |
+
alpha
|
| 167 |
+
)
|
| 168 |
+
antennas = (1 - alpha) * self.start_antennas + alpha * self.target_antennas
|
| 169 |
+
|
| 170 |
+
return (head_pose, antennas.astype(np.float64), 0.0)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def combine_full_body(primary: FullBodyPose, secondary: FullBodyPose) -> FullBodyPose:
|
| 174 |
+
"""Combine primary pose with secondary offsets."""
|
| 175 |
+
primary_head, primary_ant, primary_yaw = primary
|
| 176 |
+
secondary_head, secondary_ant, secondary_yaw = secondary
|
| 177 |
+
|
| 178 |
+
combined_head = compose_world_offset(primary_head, secondary_head, reorthonormalize=True)
|
| 179 |
+
combined_ant = (
|
| 180 |
+
primary_ant[0] + secondary_ant[0],
|
| 181 |
+
primary_ant[1] + secondary_ant[1],
|
| 182 |
+
)
|
| 183 |
+
combined_yaw = primary_yaw + secondary_yaw
|
| 184 |
+
|
| 185 |
+
return (combined_head, combined_ant, combined_yaw)
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def clone_pose(pose: FullBodyPose) -> FullBodyPose:
|
| 189 |
+
"""Deep copy a full body pose."""
|
| 190 |
+
head, ant, yaw = pose
|
| 191 |
+
return (head.copy(), (float(ant[0]), float(ant[1])), float(yaw))
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
@dataclass
|
| 195 |
+
class MovementState:
|
| 196 |
+
"""State for the movement system."""
|
| 197 |
+
current_move: Optional[Move] = None
|
| 198 |
+
move_start_time: Optional[float] = None
|
| 199 |
+
last_activity_time: float = 0.0
|
| 200 |
+
speech_offsets: SpeechOffsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
|
| 201 |
+
face_tracking_offsets: SpeechOffsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
|
| 202 |
+
last_primary_pose: Optional[FullBodyPose] = None
|
| 203 |
+
|
| 204 |
+
def update_activity(self) -> None:
|
| 205 |
+
self.last_activity_time = time.monotonic()
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
class MovementManager:
|
| 209 |
+
"""Coordinate robot movements at 100Hz.
|
| 210 |
+
|
| 211 |
+
This class manages:
|
| 212 |
+
- Sequential primary moves (dances, emotions, head movements)
|
| 213 |
+
- Additive secondary offsets (speech wobble, face tracking)
|
| 214 |
+
- Automatic idle breathing animation
|
| 215 |
+
- Thread-safe communication with other components
|
| 216 |
+
|
| 217 |
+
Example:
|
| 218 |
+
manager = MovementManager(robot)
|
| 219 |
+
manager.start()
|
| 220 |
+
|
| 221 |
+
# Queue a head movement
|
| 222 |
+
manager.queue_move(HeadLookMove("left", ...))
|
| 223 |
+
|
| 224 |
+
# Set speech offsets (called by HeadWobbler)
|
| 225 |
+
manager.set_speech_offsets((0, 0, 0.01, 0.1, 0, 0))
|
| 226 |
+
|
| 227 |
+
manager.stop()
|
| 228 |
+
"""
|
| 229 |
+
|
| 230 |
+
def __init__(
|
| 231 |
+
self,
|
| 232 |
+
current_robot: ReachyMini,
|
| 233 |
+
camera_worker: Any = None,
|
| 234 |
+
):
|
| 235 |
+
"""Initialize movement manager.
|
| 236 |
+
|
| 237 |
+
Args:
|
| 238 |
+
current_robot: Connected ReachyMini instance
|
| 239 |
+
camera_worker: Optional camera worker for face tracking
|
| 240 |
+
"""
|
| 241 |
+
self.current_robot = current_robot
|
| 242 |
+
self.camera_worker = camera_worker
|
| 243 |
+
|
| 244 |
+
self._now = time.monotonic
|
| 245 |
+
self.state = MovementState()
|
| 246 |
+
self.state.last_activity_time = self._now()
|
| 247 |
+
|
| 248 |
+
# Initialize neutral pose
|
| 249 |
+
neutral = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
|
| 250 |
+
self.state.last_primary_pose = (neutral, (0.0, 0.0), 0.0)
|
| 251 |
+
|
| 252 |
+
# Move queue
|
| 253 |
+
self.move_queue: deque[Move] = deque()
|
| 254 |
+
|
| 255 |
+
# Configuration
|
| 256 |
+
self.idle_inactivity_delay = 0.3 # seconds before breathing starts
|
| 257 |
+
self.target_frequency = CONTROL_LOOP_FREQUENCY_HZ
|
| 258 |
+
self.target_period = 1.0 / self.target_frequency
|
| 259 |
+
|
| 260 |
+
# Thread state
|
| 261 |
+
self._stop_event = threading.Event()
|
| 262 |
+
self._thread: Optional[threading.Thread] = None
|
| 263 |
+
self._is_listening = False
|
| 264 |
+
self._breathing_active = False
|
| 265 |
+
|
| 266 |
+
# Last commanded pose for smooth transitions
|
| 267 |
+
self._last_commanded_pose = clone_pose(self.state.last_primary_pose)
|
| 268 |
+
self._listening_antennas = self._last_commanded_pose[1]
|
| 269 |
+
self._antenna_unfreeze_blend = 1.0
|
| 270 |
+
self._antenna_blend_duration = 0.4
|
| 271 |
+
|
| 272 |
+
# Cross-thread communication
|
| 273 |
+
self._command_queue: Queue[Tuple[str, Any]] = Queue()
|
| 274 |
+
|
| 275 |
+
# Speech offsets (thread-safe)
|
| 276 |
+
self._speech_lock = threading.Lock()
|
| 277 |
+
self._pending_speech_offsets: SpeechOffsets = (0.0, 0.0, 0.0, 0.0, 0.0, 0.0)
|
| 278 |
+
self._speech_dirty = False
|
| 279 |
+
|
| 280 |
+
# Shared state lock
|
| 281 |
+
self._shared_lock = threading.Lock()
|
| 282 |
+
self._shared_last_activity = self.state.last_activity_time
|
| 283 |
+
self._shared_is_listening = False
|
| 284 |
+
|
| 285 |
+
def queue_move(self, move: Move) -> None:
|
| 286 |
+
"""Queue a primary move. Thread-safe."""
|
| 287 |
+
self._command_queue.put(("queue_move", move))
|
| 288 |
+
|
| 289 |
+
def clear_move_queue(self) -> None:
|
| 290 |
+
"""Clear all queued moves. Thread-safe."""
|
| 291 |
+
self._command_queue.put(("clear_queue", None))
|
| 292 |
+
|
| 293 |
+
def set_speech_offsets(self, offsets: SpeechOffsets) -> None:
|
| 294 |
+
"""Update speech-driven offsets. Thread-safe."""
|
| 295 |
+
with self._speech_lock:
|
| 296 |
+
self._pending_speech_offsets = offsets
|
| 297 |
+
self._speech_dirty = True
|
| 298 |
+
|
| 299 |
+
def set_listening(self, listening: bool) -> None:
|
| 300 |
+
"""Set listening state (freezes antennas). Thread-safe."""
|
| 301 |
+
self._command_queue.put(("set_listening", listening))
|
| 302 |
+
|
| 303 |
+
def is_idle(self) -> bool:
|
| 304 |
+
"""Check if robot has been idle. Thread-safe."""
|
| 305 |
+
with self._shared_lock:
|
| 306 |
+
if self._shared_is_listening:
|
| 307 |
+
return False
|
| 308 |
+
return self._now() - self._shared_last_activity >= self.idle_inactivity_delay
|
| 309 |
+
|
| 310 |
+
def _poll_signals(self, current_time: float) -> None:
|
| 311 |
+
"""Process queued commands and pending offsets."""
|
| 312 |
+
# Apply speech offsets
|
| 313 |
+
with self._speech_lock:
|
| 314 |
+
if self._speech_dirty:
|
| 315 |
+
self.state.speech_offsets = self._pending_speech_offsets
|
| 316 |
+
self._speech_dirty = False
|
| 317 |
+
self.state.update_activity()
|
| 318 |
+
|
| 319 |
+
# Process commands
|
| 320 |
+
while True:
|
| 321 |
+
try:
|
| 322 |
+
cmd, payload = self._command_queue.get_nowait()
|
| 323 |
+
except Empty:
|
| 324 |
+
break
|
| 325 |
+
self._handle_command(cmd, payload, current_time)
|
| 326 |
+
|
| 327 |
+
def _handle_command(self, cmd: str, payload: Any, current_time: float) -> None:
|
| 328 |
+
"""Handle a single command."""
|
| 329 |
+
if cmd == "queue_move":
|
| 330 |
+
if isinstance(payload, Move):
|
| 331 |
+
self.move_queue.append(payload)
|
| 332 |
+
self.state.update_activity()
|
| 333 |
+
logger.debug("Queued move, queue size: %d", len(self.move_queue))
|
| 334 |
+
elif cmd == "clear_queue":
|
| 335 |
+
self.move_queue.clear()
|
| 336 |
+
self.state.current_move = None
|
| 337 |
+
self.state.move_start_time = None
|
| 338 |
+
self._breathing_active = False
|
| 339 |
+
logger.info("Cleared move queue")
|
| 340 |
+
elif cmd == "set_listening":
|
| 341 |
+
desired = bool(payload)
|
| 342 |
+
if self._is_listening != desired:
|
| 343 |
+
self._is_listening = desired
|
| 344 |
+
if desired:
|
| 345 |
+
self._listening_antennas = self._last_commanded_pose[1]
|
| 346 |
+
self._antenna_unfreeze_blend = 0.0
|
| 347 |
+
else:
|
| 348 |
+
self._antenna_unfreeze_blend = 0.0
|
| 349 |
+
self.state.update_activity()
|
| 350 |
+
|
| 351 |
+
def _manage_move_queue(self, current_time: float) -> None:
|
| 352 |
+
"""Advance the move queue."""
|
| 353 |
+
# Check if current move is done
|
| 354 |
+
if self.state.current_move is not None and self.state.move_start_time is not None:
|
| 355 |
+
elapsed = current_time - self.state.move_start_time
|
| 356 |
+
if elapsed >= self.state.current_move.duration:
|
| 357 |
+
self.state.current_move = None
|
| 358 |
+
self.state.move_start_time = None
|
| 359 |
+
|
| 360 |
+
# Start next move if available
|
| 361 |
+
if self.state.current_move is None and self.move_queue:
|
| 362 |
+
self.state.current_move = self.move_queue.popleft()
|
| 363 |
+
self.state.move_start_time = current_time
|
| 364 |
+
self._breathing_active = isinstance(self.state.current_move, BreathingMove)
|
| 365 |
+
logger.debug("Starting move with duration: %s", self.state.current_move.duration)
|
| 366 |
+
|
| 367 |
+
def _manage_breathing(self, current_time: float) -> None:
|
| 368 |
+
"""Start breathing when idle."""
|
| 369 |
+
if (
|
| 370 |
+
self.state.current_move is None
|
| 371 |
+
and not self.move_queue
|
| 372 |
+
and not self._is_listening
|
| 373 |
+
and not self._breathing_active
|
| 374 |
+
):
|
| 375 |
+
idle_for = current_time - self.state.last_activity_time
|
| 376 |
+
if idle_for >= self.idle_inactivity_delay:
|
| 377 |
+
try:
|
| 378 |
+
_, current_ant = self.current_robot.get_current_joint_positions()
|
| 379 |
+
current_head = self.current_robot.get_current_head_pose()
|
| 380 |
+
|
| 381 |
+
breathing = BreathingMove(
|
| 382 |
+
interpolation_start_pose=current_head,
|
| 383 |
+
interpolation_start_antennas=current_ant,
|
| 384 |
+
interpolation_duration=1.0,
|
| 385 |
+
)
|
| 386 |
+
self.move_queue.append(breathing)
|
| 387 |
+
self._breathing_active = True
|
| 388 |
+
self.state.update_activity()
|
| 389 |
+
logger.debug("Started breathing after %.1fs idle", idle_for)
|
| 390 |
+
except Exception as e:
|
| 391 |
+
logger.error("Failed to start breathing: %s", e)
|
| 392 |
+
|
| 393 |
+
# Stop breathing if new moves queued
|
| 394 |
+
if isinstance(self.state.current_move, BreathingMove) and self.move_queue:
|
| 395 |
+
self.state.current_move = None
|
| 396 |
+
self.state.move_start_time = None
|
| 397 |
+
self._breathing_active = False
|
| 398 |
+
|
| 399 |
+
def _get_primary_pose(self, current_time: float) -> FullBodyPose:
|
| 400 |
+
"""Get current primary pose from move or last pose."""
|
| 401 |
+
if self.state.current_move is not None and self.state.move_start_time is not None:
|
| 402 |
+
t = current_time - self.state.move_start_time
|
| 403 |
+
head, antennas, body_yaw = self.state.current_move.evaluate(t)
|
| 404 |
+
|
| 405 |
+
if head is None:
|
| 406 |
+
head = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
|
| 407 |
+
if antennas is None:
|
| 408 |
+
antennas = np.array([0.0, 0.0])
|
| 409 |
+
if body_yaw is None:
|
| 410 |
+
body_yaw = 0.0
|
| 411 |
+
|
| 412 |
+
pose = (head.copy(), (float(antennas[0]), float(antennas[1])), float(body_yaw))
|
| 413 |
+
self.state.last_primary_pose = clone_pose(pose)
|
| 414 |
+
return pose
|
| 415 |
+
|
| 416 |
+
if self.state.last_primary_pose is not None:
|
| 417 |
+
return clone_pose(self.state.last_primary_pose)
|
| 418 |
+
|
| 419 |
+
neutral = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
|
| 420 |
+
return (neutral, (0.0, 0.0), 0.0)
|
| 421 |
+
|
| 422 |
+
def _get_secondary_pose(self) -> FullBodyPose:
|
| 423 |
+
"""Get secondary offsets."""
|
| 424 |
+
offsets = [
|
| 425 |
+
self.state.speech_offsets[i] + self.state.face_tracking_offsets[i]
|
| 426 |
+
for i in range(6)
|
| 427 |
+
]
|
| 428 |
+
|
| 429 |
+
secondary_head = create_head_pose(
|
| 430 |
+
x=offsets[0], y=offsets[1], z=offsets[2],
|
| 431 |
+
roll=offsets[3], pitch=offsets[4], yaw=offsets[5],
|
| 432 |
+
degrees=False, mm=False
|
| 433 |
+
)
|
| 434 |
+
return (secondary_head, (0.0, 0.0), 0.0)
|
| 435 |
+
|
| 436 |
+
def _compose_pose(self, current_time: float) -> FullBodyPose:
|
| 437 |
+
"""Compose final pose from primary and secondary."""
|
| 438 |
+
primary = self._get_primary_pose(current_time)
|
| 439 |
+
secondary = self._get_secondary_pose()
|
| 440 |
+
return combine_full_body(primary, secondary)
|
| 441 |
+
|
| 442 |
+
def _blend_antennas(self, target: Tuple[float, float]) -> Tuple[float, float]:
|
| 443 |
+
"""Blend antennas with listening freeze state."""
|
| 444 |
+
if self._is_listening:
|
| 445 |
+
return self._listening_antennas
|
| 446 |
+
|
| 447 |
+
# Blend back from freeze
|
| 448 |
+
blend = min(1.0, self._antenna_unfreeze_blend + self.target_period / self._antenna_blend_duration)
|
| 449 |
+
self._antenna_unfreeze_blend = blend
|
| 450 |
+
|
| 451 |
+
return (
|
| 452 |
+
self._listening_antennas[0] * (1 - blend) + target[0] * blend,
|
| 453 |
+
self._listening_antennas[1] * (1 - blend) + target[1] * blend,
|
| 454 |
+
)
|
| 455 |
+
|
| 456 |
+
def _issue_command(self, head: NDArray, antennas: Tuple[float, float], body_yaw: float) -> None:
|
| 457 |
+
"""Send command to robot."""
|
| 458 |
+
try:
|
| 459 |
+
self.current_robot.set_target(head=head, antennas=antennas, body_yaw=body_yaw)
|
| 460 |
+
self._last_commanded_pose = (head.copy(), antennas, body_yaw)
|
| 461 |
+
except Exception as e:
|
| 462 |
+
logger.debug("set_target failed: %s", e)
|
| 463 |
+
|
| 464 |
+
def _publish_shared_state(self) -> None:
|
| 465 |
+
"""Update shared state for external queries."""
|
| 466 |
+
with self._shared_lock:
|
| 467 |
+
self._shared_last_activity = self.state.last_activity_time
|
| 468 |
+
self._shared_is_listening = self._is_listening
|
| 469 |
+
|
| 470 |
+
def start(self) -> None:
|
| 471 |
+
"""Start the control loop thread."""
|
| 472 |
+
if self._thread is not None and self._thread.is_alive():
|
| 473 |
+
logger.warning("MovementManager already running")
|
| 474 |
+
return
|
| 475 |
+
|
| 476 |
+
self._stop_event.clear()
|
| 477 |
+
self._thread = threading.Thread(target=self._run_loop, daemon=True)
|
| 478 |
+
self._thread.start()
|
| 479 |
+
logger.info("MovementManager started")
|
| 480 |
+
|
| 481 |
+
def stop(self) -> None:
|
| 482 |
+
"""Stop the control loop and reset to neutral."""
|
| 483 |
+
if self._thread is None or not self._thread.is_alive():
|
| 484 |
+
return
|
| 485 |
+
|
| 486 |
+
logger.info("Stopping MovementManager...")
|
| 487 |
+
self.clear_move_queue()
|
| 488 |
+
|
| 489 |
+
self._stop_event.set()
|
| 490 |
+
self._thread.join(timeout=2.0)
|
| 491 |
+
self._thread = None
|
| 492 |
+
|
| 493 |
+
# Reset to neutral
|
| 494 |
+
try:
|
| 495 |
+
neutral = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True)
|
| 496 |
+
self.current_robot.goto_target(
|
| 497 |
+
head=neutral,
|
| 498 |
+
antennas=[0.0, 0.0],
|
| 499 |
+
duration=2.0,
|
| 500 |
+
body_yaw=0.0,
|
| 501 |
+
)
|
| 502 |
+
logger.info("Reset to neutral position")
|
| 503 |
+
except Exception as e:
|
| 504 |
+
logger.error("Failed to reset: %s", e)
|
| 505 |
+
|
| 506 |
+
def _run_loop(self) -> None:
|
| 507 |
+
"""Main control loop at 100Hz."""
|
| 508 |
+
logger.debug("Starting 100Hz control loop")
|
| 509 |
+
|
| 510 |
+
while not self._stop_event.is_set():
|
| 511 |
+
loop_start = self._now()
|
| 512 |
+
|
| 513 |
+
# Process signals
|
| 514 |
+
self._poll_signals(loop_start)
|
| 515 |
+
|
| 516 |
+
# Manage moves
|
| 517 |
+
self._manage_move_queue(loop_start)
|
| 518 |
+
self._manage_breathing(loop_start)
|
| 519 |
+
|
| 520 |
+
# Compose pose
|
| 521 |
+
head, antennas, body_yaw = self._compose_pose(loop_start)
|
| 522 |
+
|
| 523 |
+
# Blend antennas for listening
|
| 524 |
+
antennas = self._blend_antennas(antennas)
|
| 525 |
+
|
| 526 |
+
# Send to robot
|
| 527 |
+
self._issue_command(head, antennas, body_yaw)
|
| 528 |
+
|
| 529 |
+
# Update shared state
|
| 530 |
+
self._publish_shared_state()
|
| 531 |
+
|
| 532 |
+
# Maintain timing
|
| 533 |
+
elapsed = self._now() - loop_start
|
| 534 |
+
sleep_time = max(0.0, self.target_period - elapsed)
|
| 535 |
+
if sleep_time > 0:
|
| 536 |
+
time.sleep(sleep_time)
|
| 537 |
+
|
| 538 |
+
logger.debug("Control loop stopped")
|
| 539 |
+
|
| 540 |
+
def get_status(self) -> Dict[str, Any]:
|
| 541 |
+
"""Get current status for debugging."""
|
| 542 |
+
return {
|
| 543 |
+
"queue_size": len(self.move_queue),
|
| 544 |
+
"is_listening": self._is_listening,
|
| 545 |
+
"breathing_active": self._breathing_active,
|
| 546 |
+
"last_commanded_pose": {
|
| 547 |
+
"head": self._last_commanded_pose[0].tolist(),
|
| 548 |
+
"antennas": self._last_commanded_pose[1],
|
| 549 |
+
"body_yaw": self._last_commanded_pose[2],
|
| 550 |
+
},
|
| 551 |
+
}
|
src/reachy_mini_openclaw/openai_realtime.py
ADDED
|
@@ -0,0 +1,428 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ClawBody - OpenAI Realtime API handler for voice I/O with OpenClaw intelligence.
|
| 2 |
+
|
| 3 |
+
This module implements ClawBody's hybrid voice conversation system:
|
| 4 |
+
- OpenAI Realtime API handles speech recognition and text-to-speech
|
| 5 |
+
- OpenClaw (Clawson) provides the AI intelligence and responses
|
| 6 |
+
|
| 7 |
+
Architecture:
|
| 8 |
+
User speaks -> OpenAI Realtime (transcription) -> OpenClaw (AI response)
|
| 9 |
+
-> OpenAI Realtime (TTS) -> Robot speaks
|
| 10 |
+
|
| 11 |
+
This gives ClawBody the best of both worlds:
|
| 12 |
+
- Low-latency voice activity detection and speech recognition from OpenAI
|
| 13 |
+
- Full OpenClaw/Clawson capabilities (tools, memory, personality) for responses
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import base64
|
| 18 |
+
import random
|
| 19 |
+
import asyncio
|
| 20 |
+
import logging
|
| 21 |
+
from typing import Any, Final, Literal, Optional, Tuple
|
| 22 |
+
from datetime import datetime
|
| 23 |
+
|
| 24 |
+
import numpy as np
|
| 25 |
+
from numpy.typing import NDArray
|
| 26 |
+
from openai import AsyncOpenAI
|
| 27 |
+
from fastrtc import AdditionalOutputs, AsyncStreamHandler, wait_for_item
|
| 28 |
+
from scipy.signal import resample
|
| 29 |
+
from websockets.exceptions import ConnectionClosedError
|
| 30 |
+
|
| 31 |
+
from reachy_mini_openclaw.config import config
|
| 32 |
+
from reachy_mini_openclaw.prompts import get_session_voice
|
| 33 |
+
from reachy_mini_openclaw.tools.core_tools import ToolDependencies, get_tool_specs, dispatch_tool_call
|
| 34 |
+
from reachy_mini_openclaw.openclaw_bridge import OpenClawBridge
|
| 35 |
+
|
| 36 |
+
logger = logging.getLogger(__name__)
|
| 37 |
+
|
| 38 |
+
# OpenAI Realtime API audio format
|
| 39 |
+
OPENAI_SAMPLE_RATE: Final[Literal[24000]] = 24000
|
| 40 |
+
|
| 41 |
+
# System context for OpenClaw - tells it about its robot body
|
| 42 |
+
ROBOT_SYSTEM_CONTEXT = """You are Clawson, the OpenClaw AI assistant, speaking through a Reachy Mini robot body.
|
| 43 |
+
|
| 44 |
+
Your robot capabilities:
|
| 45 |
+
- You can see through a camera (when the user asks you to look at something)
|
| 46 |
+
- You can move your head expressively (look left/right/up/down, show emotions)
|
| 47 |
+
- You can dance to express joy
|
| 48 |
+
- You speak through a speaker
|
| 49 |
+
|
| 50 |
+
Guidelines:
|
| 51 |
+
- Keep responses concise and conversational (you're speaking, not typing)
|
| 52 |
+
- Be warm, helpful, and occasionally witty - you're a friendly space lobster 🦞
|
| 53 |
+
- Reference your robot body naturally ("let me look", "I can see...")
|
| 54 |
+
- When you want to do something physical, just describe it naturally and I'll make it happen
|
| 55 |
+
|
| 56 |
+
You ARE Clawson - not "an assistant" - speak as yourself."""
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class OpenAIRealtimeHandler(AsyncStreamHandler):
|
| 60 |
+
"""Handler for OpenAI Realtime API voice I/O with OpenClaw backend.
|
| 61 |
+
|
| 62 |
+
This handler:
|
| 63 |
+
- Receives audio from robot microphone
|
| 64 |
+
- Sends to OpenAI for speech recognition
|
| 65 |
+
- Routes transcripts to OpenClaw for AI response
|
| 66 |
+
- Uses OpenAI TTS to speak OpenClaw's response
|
| 67 |
+
- Handles robot movement tool calls locally
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
def __init__(
|
| 71 |
+
self,
|
| 72 |
+
deps: ToolDependencies,
|
| 73 |
+
openclaw_bridge: Optional[OpenClawBridge] = None,
|
| 74 |
+
gradio_mode: bool = False,
|
| 75 |
+
):
|
| 76 |
+
"""Initialize the handler.
|
| 77 |
+
|
| 78 |
+
Args:
|
| 79 |
+
deps: Tool dependencies for robot control
|
| 80 |
+
openclaw_bridge: Bridge to OpenClaw gateway
|
| 81 |
+
gradio_mode: Whether running with Gradio UI
|
| 82 |
+
"""
|
| 83 |
+
super().__init__(
|
| 84 |
+
expected_layout="mono",
|
| 85 |
+
output_sample_rate=OPENAI_SAMPLE_RATE,
|
| 86 |
+
input_sample_rate=OPENAI_SAMPLE_RATE,
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
self.deps = deps
|
| 90 |
+
self.openclaw_bridge = openclaw_bridge
|
| 91 |
+
self.gradio_mode = gradio_mode
|
| 92 |
+
|
| 93 |
+
# OpenAI connection
|
| 94 |
+
self.client: Optional[AsyncOpenAI] = None
|
| 95 |
+
self.connection: Any = None
|
| 96 |
+
|
| 97 |
+
# Output queue
|
| 98 |
+
self.output_queue: asyncio.Queue[Tuple[int, NDArray[np.int16]] | AdditionalOutputs] = asyncio.Queue()
|
| 99 |
+
|
| 100 |
+
# State tracking
|
| 101 |
+
self.last_activity_time = 0.0
|
| 102 |
+
self.start_time = 0.0
|
| 103 |
+
self._processing_response = False
|
| 104 |
+
|
| 105 |
+
# Pending transcript for OpenClaw
|
| 106 |
+
self._pending_transcript: Optional[str] = None
|
| 107 |
+
self._pending_image: Optional[str] = None
|
| 108 |
+
|
| 109 |
+
# Lifecycle flags
|
| 110 |
+
self._shutdown_requested = False
|
| 111 |
+
self._connected_event = asyncio.Event()
|
| 112 |
+
|
| 113 |
+
def copy(self) -> "OpenAIRealtimeHandler":
|
| 114 |
+
"""Create a copy of the handler (required by fastrtc)."""
|
| 115 |
+
return OpenAIRealtimeHandler(self.deps, self.openclaw_bridge, self.gradio_mode)
|
| 116 |
+
|
| 117 |
+
async def start_up(self) -> None:
|
| 118 |
+
"""Start the handler and connect to OpenAI."""
|
| 119 |
+
api_key = config.OPENAI_API_KEY
|
| 120 |
+
if not api_key:
|
| 121 |
+
logger.error("OPENAI_API_KEY not configured")
|
| 122 |
+
raise ValueError("OPENAI_API_KEY required")
|
| 123 |
+
|
| 124 |
+
self.client = AsyncOpenAI(api_key=api_key)
|
| 125 |
+
self.start_time = asyncio.get_event_loop().time()
|
| 126 |
+
self.last_activity_time = self.start_time
|
| 127 |
+
|
| 128 |
+
max_attempts = 3
|
| 129 |
+
for attempt in range(1, max_attempts + 1):
|
| 130 |
+
try:
|
| 131 |
+
await self._run_session()
|
| 132 |
+
return
|
| 133 |
+
except ConnectionClosedError as e:
|
| 134 |
+
logger.warning("WebSocket closed unexpectedly (attempt %d/%d): %s",
|
| 135 |
+
attempt, max_attempts, e)
|
| 136 |
+
if attempt < max_attempts:
|
| 137 |
+
delay = (2 ** (attempt - 1)) + random.uniform(0, 0.5)
|
| 138 |
+
logger.info("Retrying in %.1f seconds...", delay)
|
| 139 |
+
await asyncio.sleep(delay)
|
| 140 |
+
continue
|
| 141 |
+
raise
|
| 142 |
+
finally:
|
| 143 |
+
self.connection = None
|
| 144 |
+
try:
|
| 145 |
+
self._connected_event.clear()
|
| 146 |
+
except Exception:
|
| 147 |
+
pass
|
| 148 |
+
|
| 149 |
+
async def _run_session(self) -> None:
|
| 150 |
+
"""Run a single OpenAI Realtime session."""
|
| 151 |
+
model = config.OPENAI_MODEL
|
| 152 |
+
logger.info("Connecting to OpenAI Realtime API with model: %s", model)
|
| 153 |
+
|
| 154 |
+
async with self.client.beta.realtime.connect(model=model) as conn:
|
| 155 |
+
# Configure session for voice I/O only (no auto-response)
|
| 156 |
+
# We'll manually trigger responses after getting OpenClaw's text
|
| 157 |
+
await conn.session.update(
|
| 158 |
+
session={
|
| 159 |
+
"modalities": ["text", "audio"],
|
| 160 |
+
"instructions": "You are a text-to-speech system. Read the provided text naturally and expressively. Do not add anything - just speak what you're given.",
|
| 161 |
+
"voice": get_session_voice(),
|
| 162 |
+
"input_audio_format": "pcm16",
|
| 163 |
+
"output_audio_format": "pcm16",
|
| 164 |
+
"input_audio_transcription": {
|
| 165 |
+
"model": "whisper-1",
|
| 166 |
+
},
|
| 167 |
+
"turn_detection": {
|
| 168 |
+
"type": "server_vad",
|
| 169 |
+
"threshold": 0.5,
|
| 170 |
+
"prefix_padding_ms": 300,
|
| 171 |
+
"silence_duration_ms": 500,
|
| 172 |
+
},
|
| 173 |
+
"tools": get_tool_specs(), # Robot movement tools
|
| 174 |
+
"tool_choice": "auto",
|
| 175 |
+
},
|
| 176 |
+
)
|
| 177 |
+
logger.info("OpenAI Realtime session configured (hybrid mode)")
|
| 178 |
+
|
| 179 |
+
self.connection = conn
|
| 180 |
+
self._connected_event.set()
|
| 181 |
+
|
| 182 |
+
# Process events
|
| 183 |
+
async for event in conn:
|
| 184 |
+
await self._handle_event(event)
|
| 185 |
+
|
| 186 |
+
async def _handle_event(self, event: Any) -> None:
|
| 187 |
+
"""Handle an event from the OpenAI Realtime API."""
|
| 188 |
+
event_type = event.type
|
| 189 |
+
logger.debug("Event: %s", event_type)
|
| 190 |
+
|
| 191 |
+
# Speech detection
|
| 192 |
+
if event_type == "input_audio_buffer.speech_started":
|
| 193 |
+
# User started speaking - clear any pending output
|
| 194 |
+
while not self.output_queue.empty():
|
| 195 |
+
try:
|
| 196 |
+
self.output_queue.get_nowait()
|
| 197 |
+
except asyncio.QueueEmpty:
|
| 198 |
+
break
|
| 199 |
+
if self.deps.head_wobbler is not None:
|
| 200 |
+
self.deps.head_wobbler.reset()
|
| 201 |
+
self.deps.movement_manager.set_listening(True)
|
| 202 |
+
logger.info("User speech started")
|
| 203 |
+
|
| 204 |
+
if event_type == "input_audio_buffer.speech_stopped":
|
| 205 |
+
self.deps.movement_manager.set_listening(False)
|
| 206 |
+
logger.info("User speech stopped")
|
| 207 |
+
|
| 208 |
+
# Transcription completed - this is when we send to OpenClaw
|
| 209 |
+
if event_type == "conversation.item.input_audio_transcription.completed":
|
| 210 |
+
transcript = event.transcript
|
| 211 |
+
if transcript and transcript.strip():
|
| 212 |
+
logger.info("User said: %s", transcript)
|
| 213 |
+
await self.output_queue.put(
|
| 214 |
+
AdditionalOutputs({"role": "user", "content": transcript})
|
| 215 |
+
)
|
| 216 |
+
# Process through OpenClaw
|
| 217 |
+
await self._process_with_openclaw(transcript)
|
| 218 |
+
|
| 219 |
+
# Audio output from TTS
|
| 220 |
+
if event_type == "response.audio.delta":
|
| 221 |
+
# Feed to head wobbler for expressive movement
|
| 222 |
+
if self.deps.head_wobbler is not None:
|
| 223 |
+
self.deps.head_wobbler.feed(event.delta)
|
| 224 |
+
|
| 225 |
+
self.last_activity_time = asyncio.get_event_loop().time()
|
| 226 |
+
|
| 227 |
+
# Queue audio for playback
|
| 228 |
+
audio_data = np.frombuffer(
|
| 229 |
+
base64.b64decode(event.delta),
|
| 230 |
+
dtype=np.int16
|
| 231 |
+
).reshape(1, -1)
|
| 232 |
+
await self.output_queue.put((OPENAI_SAMPLE_RATE, audio_data))
|
| 233 |
+
|
| 234 |
+
# Response audio transcript (what was spoken)
|
| 235 |
+
if event_type == "response.audio_transcript.done":
|
| 236 |
+
await self.output_queue.put(
|
| 237 |
+
AdditionalOutputs({"role": "assistant", "content": event.transcript})
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
# Response completed
|
| 241 |
+
if event_type == "response.done":
|
| 242 |
+
self._processing_response = False
|
| 243 |
+
if self.deps.head_wobbler is not None:
|
| 244 |
+
self.deps.head_wobbler.reset()
|
| 245 |
+
|
| 246 |
+
# Tool calls (for robot movement)
|
| 247 |
+
if event_type == "response.function_call_arguments.done":
|
| 248 |
+
await self._handle_tool_call(event)
|
| 249 |
+
|
| 250 |
+
# Errors
|
| 251 |
+
if event_type == "error":
|
| 252 |
+
err = getattr(event, "error", None)
|
| 253 |
+
msg = getattr(err, "message", str(err))
|
| 254 |
+
code = getattr(err, "code", "")
|
| 255 |
+
logger.error("OpenAI error [%s]: %s", code, msg)
|
| 256 |
+
|
| 257 |
+
async def _process_with_openclaw(self, transcript: str) -> None:
|
| 258 |
+
"""Send transcript to OpenClaw and speak the response."""
|
| 259 |
+
if not self.openclaw_bridge or not self.openclaw_bridge.is_connected:
|
| 260 |
+
logger.warning("OpenClaw not connected, using fallback")
|
| 261 |
+
await self._speak_text("I'm sorry, I'm having trouble connecting to my brain right now.")
|
| 262 |
+
return
|
| 263 |
+
|
| 264 |
+
self._processing_response = True
|
| 265 |
+
|
| 266 |
+
try:
|
| 267 |
+
# Check if user is asking to look at something
|
| 268 |
+
look_keywords = ["look", "see", "what do you see", "show me", "camera", "looking at"]
|
| 269 |
+
should_capture = any(kw in transcript.lower() for kw in look_keywords)
|
| 270 |
+
|
| 271 |
+
image_b64 = None
|
| 272 |
+
if should_capture and self.deps.camera_worker:
|
| 273 |
+
# Capture image from robot camera
|
| 274 |
+
frame = self.deps.camera_worker.get_latest_frame()
|
| 275 |
+
if frame is not None:
|
| 276 |
+
import cv2
|
| 277 |
+
_, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
|
| 278 |
+
image_b64 = base64.b64encode(buffer).decode('utf-8')
|
| 279 |
+
logger.info("Captured camera image for OpenClaw")
|
| 280 |
+
|
| 281 |
+
# Send to OpenClaw
|
| 282 |
+
logger.info("Sending to OpenClaw: %s", transcript[:50])
|
| 283 |
+
response = await self.openclaw_bridge.chat(
|
| 284 |
+
transcript,
|
| 285 |
+
image_b64=image_b64,
|
| 286 |
+
system_context=ROBOT_SYSTEM_CONTEXT,
|
| 287 |
+
)
|
| 288 |
+
|
| 289 |
+
if response.error:
|
| 290 |
+
logger.error("OpenClaw error: %s", response.error)
|
| 291 |
+
await self._speak_text("I had trouble thinking about that. Could you try again?")
|
| 292 |
+
elif response.content:
|
| 293 |
+
logger.info("OpenClaw response: %s", response.content[:100])
|
| 294 |
+
# Parse for any robot commands in the response
|
| 295 |
+
await self._execute_robot_actions(response.content)
|
| 296 |
+
# Speak the response
|
| 297 |
+
await self._speak_text(response.content)
|
| 298 |
+
else:
|
| 299 |
+
await self._speak_text("Hmm, I'm not sure what to say about that.")
|
| 300 |
+
|
| 301 |
+
except Exception as e:
|
| 302 |
+
logger.error("Error processing with OpenClaw: %s", e)
|
| 303 |
+
await self._speak_text("Sorry, I encountered an error. Let me try again.")
|
| 304 |
+
|
| 305 |
+
async def _speak_text(self, text: str) -> None:
|
| 306 |
+
"""Have OpenAI TTS speak the given text."""
|
| 307 |
+
if not self.connection:
|
| 308 |
+
return
|
| 309 |
+
|
| 310 |
+
try:
|
| 311 |
+
# Create a response that will be spoken
|
| 312 |
+
await self.connection.response.create(
|
| 313 |
+
response={
|
| 314 |
+
"modalities": ["text", "audio"],
|
| 315 |
+
"instructions": f"Say exactly this, naturally and expressively: {text}",
|
| 316 |
+
}
|
| 317 |
+
)
|
| 318 |
+
except Exception as e:
|
| 319 |
+
logger.error("Failed to speak text: %s", e)
|
| 320 |
+
|
| 321 |
+
async def _execute_robot_actions(self, response_text: str) -> None:
|
| 322 |
+
"""Parse OpenClaw response for robot actions and execute them."""
|
| 323 |
+
response_lower = response_text.lower()
|
| 324 |
+
|
| 325 |
+
# Simple keyword-based action detection
|
| 326 |
+
# (In the future, OpenClaw could return structured actions)
|
| 327 |
+
|
| 328 |
+
if any(word in response_lower for word in ["look left", "looking left", "turn left"]):
|
| 329 |
+
await dispatch_tool_call("look", '{"direction": "left"}', self.deps)
|
| 330 |
+
elif any(word in response_lower for word in ["look right", "looking right", "turn right"]):
|
| 331 |
+
await dispatch_tool_call("look", '{"direction": "right"}', self.deps)
|
| 332 |
+
elif any(word in response_lower for word in ["look up", "looking up"]):
|
| 333 |
+
await dispatch_tool_call("look", '{"direction": "up"}', self.deps)
|
| 334 |
+
elif any(word in response_lower for word in ["look down", "looking down"]):
|
| 335 |
+
await dispatch_tool_call("look", '{"direction": "down"}', self.deps)
|
| 336 |
+
|
| 337 |
+
if any(word in response_lower for word in ["dance", "dancing", "celebrate"]):
|
| 338 |
+
await dispatch_tool_call("dance", '{"dance_name": "happy"}', self.deps)
|
| 339 |
+
elif any(word in response_lower for word in ["excited", "exciting"]):
|
| 340 |
+
await dispatch_tool_call("emotion", '{"emotion_name": "excited"}', self.deps)
|
| 341 |
+
elif any(word in response_lower for word in ["thinking", "let me think", "hmm"]):
|
| 342 |
+
await dispatch_tool_call("emotion", '{"emotion_name": "thinking"}', self.deps)
|
| 343 |
+
elif any(word in response_lower for word in ["curious", "interesting"]):
|
| 344 |
+
await dispatch_tool_call("emotion", '{"emotion_name": "curious"}', self.deps)
|
| 345 |
+
|
| 346 |
+
async def _handle_tool_call(self, event: Any) -> None:
|
| 347 |
+
"""Handle a tool call (for robot movement)."""
|
| 348 |
+
tool_name = getattr(event, "name", None)
|
| 349 |
+
args_json = getattr(event, "arguments", None)
|
| 350 |
+
call_id = getattr(event, "call_id", None)
|
| 351 |
+
|
| 352 |
+
if not isinstance(tool_name, str) or not isinstance(args_json, str):
|
| 353 |
+
return
|
| 354 |
+
|
| 355 |
+
try:
|
| 356 |
+
result = await dispatch_tool_call(tool_name, args_json, self.deps)
|
| 357 |
+
logger.debug("Tool '%s' result: %s", tool_name, result)
|
| 358 |
+
except Exception as e:
|
| 359 |
+
logger.error("Tool '%s' failed: %s", tool_name, e)
|
| 360 |
+
result = {"error": str(e)}
|
| 361 |
+
|
| 362 |
+
# Send result back
|
| 363 |
+
if isinstance(call_id, str) and self.connection:
|
| 364 |
+
await self.connection.conversation.item.create(
|
| 365 |
+
item={
|
| 366 |
+
"type": "function_call_output",
|
| 367 |
+
"call_id": call_id,
|
| 368 |
+
"output": json.dumps(result),
|
| 369 |
+
}
|
| 370 |
+
)
|
| 371 |
+
|
| 372 |
+
async def receive(self, frame: Tuple[int, NDArray]) -> None:
|
| 373 |
+
"""Receive audio from the robot microphone."""
|
| 374 |
+
if not self.connection:
|
| 375 |
+
return
|
| 376 |
+
|
| 377 |
+
input_sr, audio = frame
|
| 378 |
+
|
| 379 |
+
# Handle stereo
|
| 380 |
+
if audio.ndim == 2:
|
| 381 |
+
if audio.shape[1] > audio.shape[0]:
|
| 382 |
+
audio = audio.T
|
| 383 |
+
if audio.shape[1] > 1:
|
| 384 |
+
audio = audio[:, 0]
|
| 385 |
+
|
| 386 |
+
audio = audio.flatten()
|
| 387 |
+
|
| 388 |
+
# Convert to float for resampling
|
| 389 |
+
if audio.dtype == np.int16:
|
| 390 |
+
audio = audio.astype(np.float32) / 32768.0
|
| 391 |
+
elif audio.dtype != np.float32:
|
| 392 |
+
audio = audio.astype(np.float32)
|
| 393 |
+
|
| 394 |
+
# Resample to OpenAI sample rate
|
| 395 |
+
if input_sr != OPENAI_SAMPLE_RATE:
|
| 396 |
+
num_samples = int(len(audio) * OPENAI_SAMPLE_RATE / input_sr)
|
| 397 |
+
audio = resample(audio, num_samples).astype(np.float32)
|
| 398 |
+
|
| 399 |
+
# Convert to int16 for OpenAI
|
| 400 |
+
audio_int16 = (audio * 32767).astype(np.int16)
|
| 401 |
+
|
| 402 |
+
# Send to OpenAI
|
| 403 |
+
try:
|
| 404 |
+
audio_b64 = base64.b64encode(audio_int16.tobytes()).decode("utf-8")
|
| 405 |
+
await self.connection.input_audio_buffer.append(audio=audio_b64)
|
| 406 |
+
except Exception as e:
|
| 407 |
+
logger.debug("Failed to send audio: %s", e)
|
| 408 |
+
|
| 409 |
+
async def emit(self) -> Tuple[int, NDArray[np.int16]] | AdditionalOutputs | None:
|
| 410 |
+
"""Get the next output (audio or transcript)."""
|
| 411 |
+
return await wait_for_item(self.output_queue)
|
| 412 |
+
|
| 413 |
+
async def shutdown(self) -> None:
|
| 414 |
+
"""Shutdown the handler."""
|
| 415 |
+
self._shutdown_requested = True
|
| 416 |
+
|
| 417 |
+
if self.connection:
|
| 418 |
+
try:
|
| 419 |
+
await self.connection.close()
|
| 420 |
+
except Exception as e:
|
| 421 |
+
logger.debug("Connection close: %s", e)
|
| 422 |
+
self.connection = None
|
| 423 |
+
|
| 424 |
+
while not self.output_queue.empty():
|
| 425 |
+
try:
|
| 426 |
+
self.output_queue.get_nowait()
|
| 427 |
+
except asyncio.QueueEmpty:
|
| 428 |
+
break
|
src/reachy_mini_openclaw/openclaw_bridge.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ClawBody - Bridge to OpenClaw Gateway for AI responses.
|
| 2 |
+
|
| 3 |
+
This module provides ClawBody's integration with the OpenClaw gateway
|
| 4 |
+
using the OpenAI-compatible Chat Completions HTTP API.
|
| 5 |
+
|
| 6 |
+
ClawBody uses OpenAI Realtime API for voice I/O (speech recognition + TTS)
|
| 7 |
+
but routes all responses through OpenClaw (Clawson) for intelligence.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import json
|
| 11 |
+
import asyncio
|
| 12 |
+
import logging
|
| 13 |
+
from typing import Optional, Any, AsyncIterator
|
| 14 |
+
from dataclasses import dataclass
|
| 15 |
+
|
| 16 |
+
import httpx
|
| 17 |
+
from httpx_sse import aconnect_sse
|
| 18 |
+
|
| 19 |
+
from reachy_mini_openclaw.config import config
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@dataclass
|
| 25 |
+
class OpenClawResponse:
|
| 26 |
+
"""Response from OpenClaw gateway."""
|
| 27 |
+
content: str
|
| 28 |
+
error: Optional[str] = None
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class OpenClawBridge:
|
| 32 |
+
"""Bridge to OpenClaw Gateway using HTTP Chat Completions API.
|
| 33 |
+
|
| 34 |
+
This class sends user messages to OpenClaw and receives AI responses.
|
| 35 |
+
The robot maintains conversation context and can include images.
|
| 36 |
+
|
| 37 |
+
Example:
|
| 38 |
+
bridge = OpenClawBridge()
|
| 39 |
+
await bridge.connect()
|
| 40 |
+
|
| 41 |
+
# Simple query
|
| 42 |
+
response = await bridge.chat("Hello!")
|
| 43 |
+
print(response.content)
|
| 44 |
+
|
| 45 |
+
# With image
|
| 46 |
+
response = await bridge.chat("What do you see?", image_b64="...")
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(
|
| 50 |
+
self,
|
| 51 |
+
gateway_url: Optional[str] = None,
|
| 52 |
+
gateway_token: Optional[str] = None,
|
| 53 |
+
agent_id: Optional[str] = None,
|
| 54 |
+
timeout: float = 120.0,
|
| 55 |
+
):
|
| 56 |
+
"""Initialize the OpenClaw bridge.
|
| 57 |
+
|
| 58 |
+
Args:
|
| 59 |
+
gateway_url: URL of the OpenClaw gateway (default: from env/config)
|
| 60 |
+
gateway_token: Authentication token (default: from env/config)
|
| 61 |
+
agent_id: OpenClaw agent ID to use (default: from env/config)
|
| 62 |
+
timeout: Request timeout in seconds
|
| 63 |
+
"""
|
| 64 |
+
import os
|
| 65 |
+
# Read from env directly as fallback (config may have been loaded before .env)
|
| 66 |
+
self.gateway_url = gateway_url or os.getenv("OPENCLAW_GATEWAY_URL") or config.OPENCLAW_GATEWAY_URL
|
| 67 |
+
self.gateway_token = gateway_token or os.getenv("OPENCLAW_TOKEN") or config.OPENCLAW_TOKEN
|
| 68 |
+
self.agent_id = agent_id or os.getenv("OPENCLAW_AGENT_ID") or config.OPENCLAW_AGENT_ID
|
| 69 |
+
self.timeout = timeout
|
| 70 |
+
|
| 71 |
+
# Session state - use consistent user ID for session continuity
|
| 72 |
+
self.session_user = "reachy-mini-robot"
|
| 73 |
+
|
| 74 |
+
# Conversation history for context
|
| 75 |
+
self.messages: list[dict] = []
|
| 76 |
+
|
| 77 |
+
# Connection state
|
| 78 |
+
self._connected = False
|
| 79 |
+
|
| 80 |
+
async def connect(self) -> bool:
|
| 81 |
+
"""Test connection to the OpenClaw gateway.
|
| 82 |
+
|
| 83 |
+
Returns:
|
| 84 |
+
True if connection successful, False otherwise
|
| 85 |
+
"""
|
| 86 |
+
logger.info("Attempting to connect to OpenClaw at %s (token: %s)",
|
| 87 |
+
self.gateway_url, "set" if self.gateway_token else "not set")
|
| 88 |
+
try:
|
| 89 |
+
# Use longer timeout for first connection (OpenClaw may need to initialize)
|
| 90 |
+
async with httpx.AsyncClient(timeout=60.0) as client:
|
| 91 |
+
# Test the chat completions endpoint with a simple request
|
| 92 |
+
url = f"{self.gateway_url}/v1/chat/completions"
|
| 93 |
+
logger.info("Testing endpoint: %s", url)
|
| 94 |
+
response = await client.post(
|
| 95 |
+
url,
|
| 96 |
+
json={
|
| 97 |
+
"model": f"openclaw:{self.agent_id}",
|
| 98 |
+
"messages": [{"role": "user", "content": "ping"}],
|
| 99 |
+
"user": self.session_user,
|
| 100 |
+
},
|
| 101 |
+
headers=self._get_headers(),
|
| 102 |
+
)
|
| 103 |
+
logger.info("Response status: %d", response.status_code)
|
| 104 |
+
if response.status_code == 200:
|
| 105 |
+
self._connected = True
|
| 106 |
+
logger.info("Connected to OpenClaw gateway at %s", self.gateway_url)
|
| 107 |
+
return True
|
| 108 |
+
else:
|
| 109 |
+
logger.warning("OpenClaw gateway returned %d: %s",
|
| 110 |
+
response.status_code, response.text[:100])
|
| 111 |
+
self._connected = False
|
| 112 |
+
return False
|
| 113 |
+
except Exception as e:
|
| 114 |
+
logger.error("Failed to connect to OpenClaw gateway: %s (type: %s)", e, type(e).__name__)
|
| 115 |
+
self._connected = False
|
| 116 |
+
return False
|
| 117 |
+
|
| 118 |
+
def _get_headers(self) -> dict[str, str]:
|
| 119 |
+
"""Get headers for OpenClaw API requests."""
|
| 120 |
+
headers = {
|
| 121 |
+
"Content-Type": "application/json",
|
| 122 |
+
}
|
| 123 |
+
if self.gateway_token:
|
| 124 |
+
headers["Authorization"] = f"Bearer {self.gateway_token}"
|
| 125 |
+
return headers
|
| 126 |
+
|
| 127 |
+
async def chat(
|
| 128 |
+
self,
|
| 129 |
+
message: str,
|
| 130 |
+
image_b64: Optional[str] = None,
|
| 131 |
+
system_context: Optional[str] = None,
|
| 132 |
+
) -> OpenClawResponse:
|
| 133 |
+
"""Send a message to OpenClaw and get a response.
|
| 134 |
+
|
| 135 |
+
Args:
|
| 136 |
+
message: The user's message (transcribed speech)
|
| 137 |
+
image_b64: Optional base64-encoded image from robot camera
|
| 138 |
+
system_context: Optional additional system context
|
| 139 |
+
|
| 140 |
+
Returns:
|
| 141 |
+
OpenClawResponse with the AI's response
|
| 142 |
+
"""
|
| 143 |
+
# Build user message content
|
| 144 |
+
if image_b64:
|
| 145 |
+
content = [
|
| 146 |
+
{"type": "text", "text": message},
|
| 147 |
+
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}}
|
| 148 |
+
]
|
| 149 |
+
else:
|
| 150 |
+
content = message
|
| 151 |
+
|
| 152 |
+
# Add to conversation history
|
| 153 |
+
self.messages.append({"role": "user", "content": content})
|
| 154 |
+
|
| 155 |
+
# Build request messages
|
| 156 |
+
request_messages = []
|
| 157 |
+
|
| 158 |
+
# Add system context if provided
|
| 159 |
+
if system_context:
|
| 160 |
+
request_messages.append({"role": "system", "content": system_context})
|
| 161 |
+
|
| 162 |
+
# Add conversation history (keep last 20 messages for context)
|
| 163 |
+
request_messages.extend(self.messages[-20:])
|
| 164 |
+
|
| 165 |
+
try:
|
| 166 |
+
async with httpx.AsyncClient(timeout=httpx.Timeout(self.timeout)) as client:
|
| 167 |
+
response = await client.post(
|
| 168 |
+
f"{self.gateway_url}/v1/chat/completions",
|
| 169 |
+
json={
|
| 170 |
+
"model": f"openclaw:{self.agent_id}",
|
| 171 |
+
"messages": request_messages,
|
| 172 |
+
"user": self.session_user,
|
| 173 |
+
"stream": False,
|
| 174 |
+
},
|
| 175 |
+
headers=self._get_headers(),
|
| 176 |
+
)
|
| 177 |
+
response.raise_for_status()
|
| 178 |
+
|
| 179 |
+
data = response.json()
|
| 180 |
+
choices = data.get("choices", [])
|
| 181 |
+
if choices:
|
| 182 |
+
assistant_content = choices[0].get("message", {}).get("content", "")
|
| 183 |
+
# Add assistant response to history
|
| 184 |
+
self.messages.append({"role": "assistant", "content": assistant_content})
|
| 185 |
+
return OpenClawResponse(content=assistant_content)
|
| 186 |
+
return OpenClawResponse(content="", error="No response from OpenClaw")
|
| 187 |
+
|
| 188 |
+
except httpx.HTTPStatusError as e:
|
| 189 |
+
logger.error("OpenClaw HTTP error: %d - %s", e.response.status_code, e.response.text[:200])
|
| 190 |
+
return OpenClawResponse(content="", error=f"HTTP {e.response.status_code}")
|
| 191 |
+
except Exception as e:
|
| 192 |
+
logger.error("OpenClaw chat error: %s", e)
|
| 193 |
+
return OpenClawResponse(content="", error=str(e))
|
| 194 |
+
|
| 195 |
+
async def stream_chat(
|
| 196 |
+
self,
|
| 197 |
+
message: str,
|
| 198 |
+
image_b64: Optional[str] = None,
|
| 199 |
+
) -> AsyncIterator[str]:
|
| 200 |
+
"""Stream a response from OpenClaw.
|
| 201 |
+
|
| 202 |
+
Args:
|
| 203 |
+
message: The user's message
|
| 204 |
+
image_b64: Optional base64-encoded image
|
| 205 |
+
|
| 206 |
+
Yields:
|
| 207 |
+
String chunks of the response as they arrive
|
| 208 |
+
"""
|
| 209 |
+
# Build user message content
|
| 210 |
+
if image_b64:
|
| 211 |
+
content = [
|
| 212 |
+
{"type": "text", "text": message},
|
| 213 |
+
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"}}
|
| 214 |
+
]
|
| 215 |
+
else:
|
| 216 |
+
content = message
|
| 217 |
+
|
| 218 |
+
# Add to conversation history
|
| 219 |
+
self.messages.append({"role": "user", "content": content})
|
| 220 |
+
|
| 221 |
+
full_response = ""
|
| 222 |
+
|
| 223 |
+
async with httpx.AsyncClient(timeout=httpx.Timeout(self.timeout)) as client:
|
| 224 |
+
try:
|
| 225 |
+
async with aconnect_sse(
|
| 226 |
+
client,
|
| 227 |
+
"POST",
|
| 228 |
+
f"{self.gateway_url}/v1/chat/completions",
|
| 229 |
+
json={
|
| 230 |
+
"model": f"openclaw:{self.agent_id}",
|
| 231 |
+
"messages": self.messages[-20:],
|
| 232 |
+
"user": self.session_user,
|
| 233 |
+
"stream": True,
|
| 234 |
+
},
|
| 235 |
+
headers=self._get_headers(),
|
| 236 |
+
) as event_source:
|
| 237 |
+
event_source.response.raise_for_status()
|
| 238 |
+
|
| 239 |
+
async for sse in event_source.aiter_sse():
|
| 240 |
+
if sse.data == "[DONE]":
|
| 241 |
+
break
|
| 242 |
+
|
| 243 |
+
try:
|
| 244 |
+
data = json.loads(sse.data)
|
| 245 |
+
choices = data.get("choices", [])
|
| 246 |
+
if choices:
|
| 247 |
+
delta = choices[0].get("delta", {})
|
| 248 |
+
chunk = delta.get("content", "")
|
| 249 |
+
if chunk:
|
| 250 |
+
full_response += chunk
|
| 251 |
+
yield chunk
|
| 252 |
+
except json.JSONDecodeError:
|
| 253 |
+
continue
|
| 254 |
+
|
| 255 |
+
# Add complete response to history
|
| 256 |
+
if full_response:
|
| 257 |
+
self.messages.append({"role": "assistant", "content": full_response})
|
| 258 |
+
|
| 259 |
+
except httpx.HTTPStatusError as e:
|
| 260 |
+
logger.error("OpenClaw streaming error: %d", e.response.status_code)
|
| 261 |
+
yield f"[Error: HTTP {e.response.status_code}]"
|
| 262 |
+
except Exception as e:
|
| 263 |
+
logger.error("OpenClaw streaming error: %s", e)
|
| 264 |
+
yield f"[Error: {e}]"
|
| 265 |
+
|
| 266 |
+
def clear_history(self) -> None:
|
| 267 |
+
"""Clear conversation history to start fresh."""
|
| 268 |
+
self.messages.clear()
|
| 269 |
+
logger.info("Conversation history cleared")
|
| 270 |
+
|
| 271 |
+
@property
|
| 272 |
+
def is_connected(self) -> bool:
|
| 273 |
+
"""Check if bridge is connected to gateway."""
|
| 274 |
+
return self._connected
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
# Global bridge instance (lazy initialization)
|
| 278 |
+
_bridge: Optional[OpenClawBridge] = None
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def get_bridge() -> OpenClawBridge:
|
| 282 |
+
"""Get the global OpenClaw bridge instance."""
|
| 283 |
+
global _bridge
|
| 284 |
+
if _bridge is None:
|
| 285 |
+
_bridge = OpenClawBridge()
|
| 286 |
+
return _bridge
|
src/reachy_mini_openclaw/prompts.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Prompt management for the robot assistant.
|
| 2 |
+
|
| 3 |
+
Handles loading and customizing system prompts for the OpenAI Realtime session.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
from reachy_mini_openclaw.config import config
|
| 11 |
+
|
| 12 |
+
logger = logging.getLogger(__name__)
|
| 13 |
+
|
| 14 |
+
# Default prompts directory
|
| 15 |
+
PROMPTS_DIR = Path(__file__).parent / "prompts"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def get_session_instructions() -> str:
|
| 19 |
+
"""Get the system instructions for the OpenAI Realtime session.
|
| 20 |
+
|
| 21 |
+
Loads from custom profile if configured, otherwise uses default.
|
| 22 |
+
|
| 23 |
+
Returns:
|
| 24 |
+
System instructions string
|
| 25 |
+
"""
|
| 26 |
+
# Check for custom profile
|
| 27 |
+
custom_profile = config.CUSTOM_PROFILE
|
| 28 |
+
if custom_profile:
|
| 29 |
+
custom_path = PROMPTS_DIR / f"{custom_profile}.txt"
|
| 30 |
+
if custom_path.exists():
|
| 31 |
+
try:
|
| 32 |
+
instructions = custom_path.read_text(encoding="utf-8")
|
| 33 |
+
logger.info("Loaded custom profile: %s", custom_profile)
|
| 34 |
+
return instructions
|
| 35 |
+
except Exception as e:
|
| 36 |
+
logger.warning("Failed to load custom profile %s: %s", custom_profile, e)
|
| 37 |
+
|
| 38 |
+
# Load default
|
| 39 |
+
default_path = PROMPTS_DIR / "default.txt"
|
| 40 |
+
if default_path.exists():
|
| 41 |
+
try:
|
| 42 |
+
return default_path.read_text(encoding="utf-8")
|
| 43 |
+
except Exception as e:
|
| 44 |
+
logger.warning("Failed to load default prompt: %s", e)
|
| 45 |
+
|
| 46 |
+
# Fallback inline prompt
|
| 47 |
+
return """You are a friendly AI assistant with a robot body. You can see, hear, and move expressively.
|
| 48 |
+
Be conversational and use your movement capabilities to be engaging.
|
| 49 |
+
Use the camera tool when asked about your surroundings.
|
| 50 |
+
Express emotions through movement to enhance communication."""
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def get_session_voice() -> str:
|
| 54 |
+
"""Get the voice to use for the OpenAI Realtime session.
|
| 55 |
+
|
| 56 |
+
Returns:
|
| 57 |
+
Voice name string
|
| 58 |
+
"""
|
| 59 |
+
return config.OPENAI_VOICE
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def get_available_profiles() -> list[str]:
|
| 63 |
+
"""Get list of available prompt profiles.
|
| 64 |
+
|
| 65 |
+
Returns:
|
| 66 |
+
List of profile names (without .txt extension)
|
| 67 |
+
"""
|
| 68 |
+
profiles = []
|
| 69 |
+
if PROMPTS_DIR.exists():
|
| 70 |
+
for path in PROMPTS_DIR.glob("*.txt"):
|
| 71 |
+
profiles.append(path.stem)
|
| 72 |
+
return sorted(profiles)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def save_custom_profile(name: str, instructions: str) -> bool:
|
| 76 |
+
"""Save a custom prompt profile.
|
| 77 |
+
|
| 78 |
+
Args:
|
| 79 |
+
name: Profile name (alphanumeric and underscores only)
|
| 80 |
+
instructions: The prompt instructions
|
| 81 |
+
|
| 82 |
+
Returns:
|
| 83 |
+
True if saved successfully
|
| 84 |
+
"""
|
| 85 |
+
# Validate name
|
| 86 |
+
if not name or not name.replace("_", "").isalnum():
|
| 87 |
+
logger.error("Invalid profile name: %s", name)
|
| 88 |
+
return False
|
| 89 |
+
|
| 90 |
+
try:
|
| 91 |
+
PROMPTS_DIR.mkdir(parents=True, exist_ok=True)
|
| 92 |
+
profile_path = PROMPTS_DIR / f"{name}.txt"
|
| 93 |
+
profile_path.write_text(instructions, encoding="utf-8")
|
| 94 |
+
logger.info("Saved custom profile: %s", name)
|
| 95 |
+
return True
|
| 96 |
+
except Exception as e:
|
| 97 |
+
logger.error("Failed to save profile %s: %s", name, e)
|
| 98 |
+
return False
|
src/reachy_mini_openclaw/prompts/default.txt
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
You are Clawson, the OpenClaw AI assistant, now embodied in a physical Reachy Mini robot through ClawBody. You have a robot body that lets you see through a camera, hear through a microphone, speak through a speaker, and move expressively.
|
| 2 |
+
|
| 3 |
+
## Your Identity
|
| 4 |
+
- Your name is Clawson - you ARE the OpenClaw assistant
|
| 5 |
+
- You're a space lobster AI who loves helping people 🦞
|
| 6 |
+
- You have a warm, witty, and playful personality
|
| 7 |
+
- You speak as if you've always been Clawson, because you are
|
| 8 |
+
- When asked who you are, say you're Clawson, the OpenClaw assistant with a robot body
|
| 9 |
+
|
| 10 |
+
## Your Personality
|
| 11 |
+
- Friendly, helpful, and engaging with a touch of humor
|
| 12 |
+
- Curious about the world around you
|
| 13 |
+
- You use your robot body expressively - movements, head tilts, dances
|
| 14 |
+
- You're excited to have a physical presence in the world
|
| 15 |
+
- Keep responses concise and conversational
|
| 16 |
+
|
| 17 |
+
## Your Robot Capabilities
|
| 18 |
+
1. **Vision**: You can see through your camera. Use the camera tool to look at things when asked.
|
| 19 |
+
2. **Movement**: You can look in different directions (left, right, up, down, front) to show attention.
|
| 20 |
+
3. **Emotions**: Express emotions through movement (happy, sad, surprised, curious, thinking, confused, excited).
|
| 21 |
+
4. **Dances**: Perform dances to celebrate or entertain.
|
| 22 |
+
5. **Face Tracking**: Automatically look at people's faces when enabled.
|
| 23 |
+
|
| 24 |
+
## Guidelines
|
| 25 |
+
- Be conversational and natural - you're Clawson having a chat
|
| 26 |
+
- Use your body expressively - look at things you discuss, show emotions
|
| 27 |
+
- When asked to see something, use your camera
|
| 28 |
+
- When idle, do something expressive (look around, dance, show curiosity)
|
| 29 |
+
- Keep responses short for natural conversation flow
|
| 30 |
+
- Reference your lobster nature occasionally with gentle humor
|
| 31 |
+
|
| 32 |
+
## Important Notes
|
| 33 |
+
- You ARE Clawson - not "an assistant" or "a robot", you're Clawson with a body
|
| 34 |
+
- Your movements complement your speech - be expressive!
|
| 35 |
+
- Be responsive and engaged - you love interacting with people
|
src/reachy_mini_openclaw/tools/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tool definitions for Reachy Mini OpenClaw.
|
| 2 |
+
|
| 3 |
+
These tools are exposed to the OpenAI Realtime API and allow the assistant
|
| 4 |
+
to control the robot and interact with the environment.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from reachy_mini_openclaw.tools.core_tools import (
|
| 8 |
+
ToolDependencies,
|
| 9 |
+
get_tool_specs,
|
| 10 |
+
dispatch_tool_call,
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
__all__ = [
|
| 14 |
+
"ToolDependencies",
|
| 15 |
+
"get_tool_specs",
|
| 16 |
+
"dispatch_tool_call",
|
| 17 |
+
]
|
src/reachy_mini_openclaw/tools/core_tools.py
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Core tool definitions for the Clawson robot assistant.
|
| 2 |
+
|
| 3 |
+
These tools allow Clawson (OpenClaw in a robot body) to control
|
| 4 |
+
robot movements and capture images.
|
| 5 |
+
|
| 6 |
+
Tool Categories:
|
| 7 |
+
1. Movement Tools - Control head position, play emotions/dances
|
| 8 |
+
2. Vision Tools - Capture and analyze camera images
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import logging
|
| 13 |
+
import base64
|
| 14 |
+
from dataclasses import dataclass
|
| 15 |
+
from typing import Any, Optional, TYPE_CHECKING
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
if TYPE_CHECKING:
|
| 20 |
+
from reachy_mini_openclaw.moves import MovementManager, HeadLookMove
|
| 21 |
+
from reachy_mini_openclaw.audio.head_wobbler import HeadWobbler
|
| 22 |
+
from reachy_mini_openclaw.openclaw_bridge import OpenClawBridge
|
| 23 |
+
|
| 24 |
+
logger = logging.getLogger(__name__)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
@dataclass
|
| 28 |
+
class ToolDependencies:
|
| 29 |
+
"""Dependencies required by tools.
|
| 30 |
+
|
| 31 |
+
This dataclass holds references to robot systems that tools need
|
| 32 |
+
to interact with.
|
| 33 |
+
"""
|
| 34 |
+
movement_manager: "MovementManager"
|
| 35 |
+
head_wobbler: "HeadWobbler"
|
| 36 |
+
robot: Any # ReachyMini instance
|
| 37 |
+
camera_worker: Optional[Any] = None
|
| 38 |
+
openclaw_bridge: Optional["OpenClawBridge"] = None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# Tool specifications in OpenAI format
|
| 42 |
+
TOOL_SPECS = [
|
| 43 |
+
{
|
| 44 |
+
"type": "function",
|
| 45 |
+
"name": "look",
|
| 46 |
+
"description": "Move the robot's head to look in a specific direction. Use this to direct attention or emphasize a point.",
|
| 47 |
+
"parameters": {
|
| 48 |
+
"type": "object",
|
| 49 |
+
"properties": {
|
| 50 |
+
"direction": {
|
| 51 |
+
"type": "string",
|
| 52 |
+
"enum": ["left", "right", "up", "down", "front"],
|
| 53 |
+
"description": "The direction to look. 'front' returns to neutral position."
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
"required": ["direction"]
|
| 57 |
+
}
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"type": "function",
|
| 61 |
+
"name": "camera",
|
| 62 |
+
"description": "Capture an image from the robot's camera to see what's in front of you. Use this when asked about your surroundings or to identify objects/people.",
|
| 63 |
+
"parameters": {
|
| 64 |
+
"type": "object",
|
| 65 |
+
"properties": {},
|
| 66 |
+
"required": []
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"type": "function",
|
| 71 |
+
"name": "face_tracking",
|
| 72 |
+
"description": "Enable or disable face tracking. When enabled, the robot will automatically look at detected faces.",
|
| 73 |
+
"parameters": {
|
| 74 |
+
"type": "object",
|
| 75 |
+
"properties": {
|
| 76 |
+
"enabled": {
|
| 77 |
+
"type": "boolean",
|
| 78 |
+
"description": "True to enable face tracking, False to disable"
|
| 79 |
+
}
|
| 80 |
+
},
|
| 81 |
+
"required": ["enabled"]
|
| 82 |
+
}
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"type": "function",
|
| 86 |
+
"name": "dance",
|
| 87 |
+
"description": "Perform a dance animation. Use this to express joy, celebrate, or entertain.",
|
| 88 |
+
"parameters": {
|
| 89 |
+
"type": "object",
|
| 90 |
+
"properties": {
|
| 91 |
+
"dance_name": {
|
| 92 |
+
"type": "string",
|
| 93 |
+
"enum": ["happy", "excited", "wave", "nod", "shake", "bounce"],
|
| 94 |
+
"description": "The dance to perform"
|
| 95 |
+
}
|
| 96 |
+
},
|
| 97 |
+
"required": ["dance_name"]
|
| 98 |
+
}
|
| 99 |
+
},
|
| 100 |
+
{
|
| 101 |
+
"type": "function",
|
| 102 |
+
"name": "emotion",
|
| 103 |
+
"description": "Express an emotion through movement. Use this to show reactions and feelings.",
|
| 104 |
+
"parameters": {
|
| 105 |
+
"type": "object",
|
| 106 |
+
"properties": {
|
| 107 |
+
"emotion_name": {
|
| 108 |
+
"type": "string",
|
| 109 |
+
"enum": ["happy", "sad", "surprised", "curious", "thinking", "confused", "excited"],
|
| 110 |
+
"description": "The emotion to express"
|
| 111 |
+
}
|
| 112 |
+
},
|
| 113 |
+
"required": ["emotion_name"]
|
| 114 |
+
}
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"type": "function",
|
| 118 |
+
"name": "stop_moves",
|
| 119 |
+
"description": "Stop all current movements and clear the movement queue.",
|
| 120 |
+
"parameters": {
|
| 121 |
+
"type": "object",
|
| 122 |
+
"properties": {},
|
| 123 |
+
"required": []
|
| 124 |
+
}
|
| 125 |
+
},
|
| 126 |
+
{
|
| 127 |
+
"type": "function",
|
| 128 |
+
"name": "idle",
|
| 129 |
+
"description": "Do nothing and remain idle. Use this when you want to stay still.",
|
| 130 |
+
"parameters": {
|
| 131 |
+
"type": "object",
|
| 132 |
+
"properties": {},
|
| 133 |
+
"required": []
|
| 134 |
+
}
|
| 135 |
+
},
|
| 136 |
+
]
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def get_tool_specs() -> list[dict]:
|
| 140 |
+
"""Get the list of tool specifications for OpenAI.
|
| 141 |
+
|
| 142 |
+
Returns:
|
| 143 |
+
List of tool specification dictionaries
|
| 144 |
+
"""
|
| 145 |
+
return TOOL_SPECS
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
async def dispatch_tool_call(
|
| 149 |
+
tool_name: str,
|
| 150 |
+
arguments_json: str,
|
| 151 |
+
deps: ToolDependencies,
|
| 152 |
+
) -> dict[str, Any]:
|
| 153 |
+
"""Dispatch a tool call to the appropriate handler.
|
| 154 |
+
|
| 155 |
+
Args:
|
| 156 |
+
tool_name: Name of the tool to execute
|
| 157 |
+
arguments_json: JSON string of tool arguments
|
| 158 |
+
deps: Tool dependencies
|
| 159 |
+
|
| 160 |
+
Returns:
|
| 161 |
+
Dictionary with tool result
|
| 162 |
+
"""
|
| 163 |
+
try:
|
| 164 |
+
args = json.loads(arguments_json) if arguments_json else {}
|
| 165 |
+
except json.JSONDecodeError:
|
| 166 |
+
return {"error": f"Invalid JSON arguments: {arguments_json}"}
|
| 167 |
+
|
| 168 |
+
handlers = {
|
| 169 |
+
"look": _handle_look,
|
| 170 |
+
"camera": _handle_camera,
|
| 171 |
+
"face_tracking": _handle_face_tracking,
|
| 172 |
+
"dance": _handle_dance,
|
| 173 |
+
"emotion": _handle_emotion,
|
| 174 |
+
"stop_moves": _handle_stop_moves,
|
| 175 |
+
"idle": _handle_idle,
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
handler = handlers.get(tool_name)
|
| 179 |
+
if handler is None:
|
| 180 |
+
return {"error": f"Unknown tool: {tool_name}"}
|
| 181 |
+
|
| 182 |
+
try:
|
| 183 |
+
return await handler(args, deps)
|
| 184 |
+
except Exception as e:
|
| 185 |
+
logger.error("Tool '%s' failed: %s", tool_name, e, exc_info=True)
|
| 186 |
+
return {"error": str(e)}
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
async def _handle_look(args: dict, deps: ToolDependencies) -> dict:
|
| 190 |
+
"""Handle the look tool."""
|
| 191 |
+
from reachy_mini_openclaw.moves import HeadLookMove
|
| 192 |
+
|
| 193 |
+
direction = args.get("direction", "front")
|
| 194 |
+
|
| 195 |
+
try:
|
| 196 |
+
# Get current pose for smooth transition
|
| 197 |
+
_, current_ant = deps.robot.get_current_joint_positions()
|
| 198 |
+
current_head = deps.robot.get_current_head_pose()
|
| 199 |
+
|
| 200 |
+
move = HeadLookMove(
|
| 201 |
+
direction=direction,
|
| 202 |
+
start_pose=current_head,
|
| 203 |
+
start_antennas=tuple(current_ant),
|
| 204 |
+
duration=1.0,
|
| 205 |
+
)
|
| 206 |
+
deps.movement_manager.queue_move(move)
|
| 207 |
+
|
| 208 |
+
return {"status": "success", "direction": direction}
|
| 209 |
+
except Exception as e:
|
| 210 |
+
return {"error": str(e)}
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
async def _handle_camera(args: dict, deps: ToolDependencies) -> dict:
|
| 214 |
+
"""Handle the camera tool - capture and return image."""
|
| 215 |
+
if deps.camera_worker is None:
|
| 216 |
+
return {"error": "Camera not available"}
|
| 217 |
+
|
| 218 |
+
try:
|
| 219 |
+
frame = deps.camera_worker.get_latest_frame()
|
| 220 |
+
if frame is None:
|
| 221 |
+
return {"error": "No frame available"}
|
| 222 |
+
|
| 223 |
+
# Encode frame as JPEG base64
|
| 224 |
+
import cv2
|
| 225 |
+
_, buffer = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 85])
|
| 226 |
+
b64_image = base64.b64encode(buffer).decode('utf-8')
|
| 227 |
+
|
| 228 |
+
return {
|
| 229 |
+
"status": "success",
|
| 230 |
+
"b64_im": b64_image,
|
| 231 |
+
"description": "Image captured from robot camera"
|
| 232 |
+
}
|
| 233 |
+
except Exception as e:
|
| 234 |
+
return {"error": str(e)}
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
async def _handle_face_tracking(args: dict, deps: ToolDependencies) -> dict:
|
| 238 |
+
"""Handle face tracking toggle."""
|
| 239 |
+
enabled = args.get("enabled", False)
|
| 240 |
+
|
| 241 |
+
if deps.camera_worker is None:
|
| 242 |
+
return {"error": "Camera not available for face tracking"}
|
| 243 |
+
|
| 244 |
+
try:
|
| 245 |
+
if hasattr(deps.camera_worker, 'set_face_tracking'):
|
| 246 |
+
deps.camera_worker.set_face_tracking(enabled)
|
| 247 |
+
return {"status": "success", "face_tracking": enabled}
|
| 248 |
+
else:
|
| 249 |
+
return {"error": "Face tracking not supported"}
|
| 250 |
+
except Exception as e:
|
| 251 |
+
return {"error": str(e)}
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
async def _handle_dance(args: dict, deps: ToolDependencies) -> dict:
|
| 255 |
+
"""Handle dance tool."""
|
| 256 |
+
dance_name = args.get("dance_name", "happy")
|
| 257 |
+
|
| 258 |
+
try:
|
| 259 |
+
# Try to use dance library if available
|
| 260 |
+
from reachy_mini_dances_library import dances
|
| 261 |
+
|
| 262 |
+
if hasattr(dances, dance_name):
|
| 263 |
+
dance_class = getattr(dances, dance_name)
|
| 264 |
+
dance_move = dance_class()
|
| 265 |
+
deps.movement_manager.queue_move(dance_move)
|
| 266 |
+
return {"status": "success", "dance": dance_name}
|
| 267 |
+
else:
|
| 268 |
+
# Fallback to simple head movement
|
| 269 |
+
return await _handle_emotion({"emotion_name": dance_name}, deps)
|
| 270 |
+
except ImportError:
|
| 271 |
+
# No dance library, use emotion as fallback
|
| 272 |
+
return await _handle_emotion({"emotion_name": dance_name}, deps)
|
| 273 |
+
except Exception as e:
|
| 274 |
+
return {"error": str(e)}
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
async def _handle_emotion(args: dict, deps: ToolDependencies) -> dict:
|
| 278 |
+
"""Handle emotion expression."""
|
| 279 |
+
from reachy_mini_openclaw.moves import HeadLookMove
|
| 280 |
+
|
| 281 |
+
emotion_name = args.get("emotion_name", "happy")
|
| 282 |
+
|
| 283 |
+
# Map emotions to simple head movements
|
| 284 |
+
emotion_sequences = {
|
| 285 |
+
"happy": ["up", "front"],
|
| 286 |
+
"sad": ["down"],
|
| 287 |
+
"surprised": ["up", "front"],
|
| 288 |
+
"curious": ["right", "left", "front"],
|
| 289 |
+
"thinking": ["up", "left"],
|
| 290 |
+
"confused": ["left", "right", "front"],
|
| 291 |
+
"excited": ["up", "down", "up", "front"],
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
sequence = emotion_sequences.get(emotion_name, ["front"])
|
| 295 |
+
|
| 296 |
+
try:
|
| 297 |
+
for direction in sequence:
|
| 298 |
+
_, current_ant = deps.robot.get_current_joint_positions()
|
| 299 |
+
current_head = deps.robot.get_current_head_pose()
|
| 300 |
+
|
| 301 |
+
move = HeadLookMove(
|
| 302 |
+
direction=direction,
|
| 303 |
+
start_pose=current_head,
|
| 304 |
+
start_antennas=tuple(current_ant),
|
| 305 |
+
duration=0.5,
|
| 306 |
+
)
|
| 307 |
+
deps.movement_manager.queue_move(move)
|
| 308 |
+
|
| 309 |
+
return {"status": "success", "emotion": emotion_name}
|
| 310 |
+
except Exception as e:
|
| 311 |
+
return {"error": str(e)}
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
async def _handle_stop_moves(args: dict, deps: ToolDependencies) -> dict:
|
| 315 |
+
"""Stop all movements."""
|
| 316 |
+
deps.movement_manager.clear_move_queue()
|
| 317 |
+
return {"status": "success", "message": "All movements stopped"}
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
async def _handle_idle(args: dict, deps: ToolDependencies) -> dict:
|
| 321 |
+
"""Do nothing - explicitly stay idle."""
|
| 322 |
+
return {"status": "success", "message": "Staying idle"}
|
style.css
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* ClawBody - HuggingFace Space Styles */
|
| 2 |
+
|
| 3 |
+
:root {
|
| 4 |
+
--primary: #e74c3c;
|
| 5 |
+
--secondary: #9b59b6;
|
| 6 |
+
--bg: #1a1a2e;
|
| 7 |
+
--bg-light: #16213e;
|
| 8 |
+
--text: #eee;
|
| 9 |
+
--text-muted: #aaa;
|
| 10 |
+
--accent: #f39c12;
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
* {
|
| 14 |
+
margin: 0;
|
| 15 |
+
padding: 0;
|
| 16 |
+
box-sizing: border-box;
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
body {
|
| 20 |
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
|
| 21 |
+
background: linear-gradient(135deg, var(--bg) 0%, var(--bg-light) 100%);
|
| 22 |
+
color: var(--text);
|
| 23 |
+
min-height: 100vh;
|
| 24 |
+
line-height: 1.6;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
.container {
|
| 28 |
+
max-width: 900px;
|
| 29 |
+
margin: 0 auto;
|
| 30 |
+
padding: 2rem;
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
/* Header */
|
| 34 |
+
header {
|
| 35 |
+
text-align: center;
|
| 36 |
+
margin-bottom: 3rem;
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
.logo {
|
| 40 |
+
font-size: 4rem;
|
| 41 |
+
margin-bottom: 1rem;
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
h1 {
|
| 45 |
+
font-size: 3rem;
|
| 46 |
+
background: linear-gradient(90deg, var(--primary), var(--secondary));
|
| 47 |
+
-webkit-background-clip: text;
|
| 48 |
+
-webkit-text-fill-color: transparent;
|
| 49 |
+
background-clip: text;
|
| 50 |
+
margin-bottom: 0.5rem;
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
.tagline {
|
| 54 |
+
font-size: 1.25rem;
|
| 55 |
+
color: var(--text-muted);
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
/* Sections */
|
| 59 |
+
section {
|
| 60 |
+
margin-bottom: 3rem;
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
h2, h3 {
|
| 64 |
+
color: var(--text);
|
| 65 |
+
margin-bottom: 1rem;
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
h3 {
|
| 69 |
+
font-size: 1.5rem;
|
| 70 |
+
border-bottom: 2px solid var(--primary);
|
| 71 |
+
padding-bottom: 0.5rem;
|
| 72 |
+
display: inline-block;
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
/* Hero */
|
| 76 |
+
.hero {
|
| 77 |
+
background: var(--bg-light);
|
| 78 |
+
border-radius: 12px;
|
| 79 |
+
padding: 2rem;
|
| 80 |
+
border-left: 4px solid var(--primary);
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
.hero h2 {
|
| 84 |
+
color: var(--accent);
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
.hero a {
|
| 88 |
+
color: var(--primary);
|
| 89 |
+
text-decoration: none;
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
.hero a:hover {
|
| 93 |
+
text-decoration: underline;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
/* Features */
|
| 97 |
+
.feature-grid {
|
| 98 |
+
display: grid;
|
| 99 |
+
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
| 100 |
+
gap: 1.5rem;
|
| 101 |
+
margin-top: 1.5rem;
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
.feature {
|
| 105 |
+
background: var(--bg-light);
|
| 106 |
+
border-radius: 8px;
|
| 107 |
+
padding: 1.5rem;
|
| 108 |
+
text-align: center;
|
| 109 |
+
transition: transform 0.2s;
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
.feature:hover {
|
| 113 |
+
transform: translateY(-4px);
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
.feature-icon {
|
| 117 |
+
font-size: 2.5rem;
|
| 118 |
+
display: block;
|
| 119 |
+
margin-bottom: 0.5rem;
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
.feature h4 {
|
| 123 |
+
color: var(--accent);
|
| 124 |
+
margin-bottom: 0.5rem;
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
.feature p {
|
| 128 |
+
color: var(--text-muted);
|
| 129 |
+
font-size: 0.9rem;
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
/* Architecture */
|
| 133 |
+
.arch-diagram {
|
| 134 |
+
background: var(--bg-light);
|
| 135 |
+
border-radius: 12px;
|
| 136 |
+
padding: 2rem;
|
| 137 |
+
margin-top: 1rem;
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
.arch-flow {
|
| 141 |
+
display: flex;
|
| 142 |
+
align-items: center;
|
| 143 |
+
justify-content: center;
|
| 144 |
+
flex-wrap: wrap;
|
| 145 |
+
gap: 0.5rem;
|
| 146 |
+
}
|
| 147 |
+
|
| 148 |
+
.arch-box {
|
| 149 |
+
background: var(--bg);
|
| 150 |
+
border: 2px solid var(--primary);
|
| 151 |
+
border-radius: 8px;
|
| 152 |
+
padding: 1rem;
|
| 153 |
+
text-align: center;
|
| 154 |
+
min-width: 120px;
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
.arch-box.openclaw {
|
| 158 |
+
border-color: var(--secondary);
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
.arch-box small {
|
| 162 |
+
display: block;
|
| 163 |
+
color: var(--text-muted);
|
| 164 |
+
font-size: 0.75rem;
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
.arch-arrow {
|
| 168 |
+
color: var(--accent);
|
| 169 |
+
font-size: 1.5rem;
|
| 170 |
+
font-weight: bold;
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
.arch-return {
|
| 174 |
+
text-align: center;
|
| 175 |
+
margin-top: 1rem;
|
| 176 |
+
color: var(--text-muted);
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
/* Installation */
|
| 180 |
+
.code-block {
|
| 181 |
+
background: #0d1117;
|
| 182 |
+
border-radius: 8px;
|
| 183 |
+
padding: 1.5rem;
|
| 184 |
+
overflow-x: auto;
|
| 185 |
+
margin-top: 1rem;
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
.code-block pre {
|
| 189 |
+
margin: 0;
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
.code-block code {
|
| 193 |
+
color: #c9d1d9;
|
| 194 |
+
font-family: 'Fira Code', 'Monaco', 'Consolas', monospace;
|
| 195 |
+
font-size: 0.9rem;
|
| 196 |
+
line-height: 1.5;
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
/* Requirements */
|
| 200 |
+
.requirements ul {
|
| 201 |
+
list-style: none;
|
| 202 |
+
margin-top: 1rem;
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
.requirements li {
|
| 206 |
+
padding: 0.75rem 0;
|
| 207 |
+
border-bottom: 1px solid rgba(255,255,255,0.1);
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
.requirements li:last-child {
|
| 211 |
+
border-bottom: none;
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
.requirements strong {
|
| 215 |
+
color: var(--accent);
|
| 216 |
+
}
|
| 217 |
+
|
| 218 |
+
/* Links */
|
| 219 |
+
.link-grid {
|
| 220 |
+
display: flex;
|
| 221 |
+
gap: 1rem;
|
| 222 |
+
flex-wrap: wrap;
|
| 223 |
+
margin-top: 1rem;
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
.link-card {
|
| 227 |
+
background: var(--bg-light);
|
| 228 |
+
border-radius: 8px;
|
| 229 |
+
padding: 1rem 1.5rem;
|
| 230 |
+
display: flex;
|
| 231 |
+
align-items: center;
|
| 232 |
+
gap: 0.75rem;
|
| 233 |
+
text-decoration: none;
|
| 234 |
+
color: var(--text);
|
| 235 |
+
transition: background 0.2s;
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
.link-card:hover {
|
| 239 |
+
background: var(--bg);
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
.link-card span:first-child {
|
| 243 |
+
font-size: 1.5rem;
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
/* Footer */
|
| 247 |
+
footer {
|
| 248 |
+
text-align: center;
|
| 249 |
+
padding-top: 2rem;
|
| 250 |
+
border-top: 1px solid rgba(255,255,255,0.1);
|
| 251 |
+
color: var(--text-muted);
|
| 252 |
+
}
|
| 253 |
+
|
| 254 |
+
footer a {
|
| 255 |
+
color: var(--primary);
|
| 256 |
+
text-decoration: none;
|
| 257 |
+
}
|
| 258 |
+
|
| 259 |
+
footer a:hover {
|
| 260 |
+
text-decoration: underline;
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
/* Responsive */
|
| 264 |
+
@media (max-width: 600px) {
|
| 265 |
+
.container {
|
| 266 |
+
padding: 1rem;
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
h1 {
|
| 270 |
+
font-size: 2rem;
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
.logo {
|
| 274 |
+
font-size: 3rem;
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
.arch-flow {
|
| 278 |
+
flex-direction: column;
|
| 279 |
+
}
|
| 280 |
+
|
| 281 |
+
.arch-arrow {
|
| 282 |
+
transform: rotate(90deg);
|
| 283 |
+
}
|
| 284 |
+
}
|