Spaces:
Paused
Paused
Upload 581 files
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .gitattributes +5 -0
- Swabble/.github/workflows/ci.yml +54 -0
- Swabble/.gitignore +33 -0
- Swabble/.swiftformat +8 -0
- Swabble/.swiftlint.yml +43 -0
- Swabble/CHANGELOG.md +11 -0
- Swabble/LICENSE +21 -0
- Swabble/Package.resolved +69 -0
- Swabble/Package.swift +55 -0
- Swabble/README.md +111 -0
- Swabble/Sources/SwabbleCore/Config/Config.swift +77 -0
- Swabble/Sources/SwabbleCore/Hooks/HookExecutor.swift +75 -0
- Swabble/Sources/SwabbleCore/Speech/BufferConverter.swift +50 -0
- Swabble/Sources/SwabbleCore/Speech/SpeechPipeline.swift +114 -0
- Swabble/Sources/SwabbleCore/Support/AttributedString+Sentences.swift +62 -0
- Swabble/Sources/SwabbleCore/Support/Logging.swift +41 -0
- Swabble/Sources/SwabbleCore/Support/OutputFormat.swift +45 -0
- Swabble/Sources/SwabbleCore/Support/TranscriptsStore.swift +45 -0
- Swabble/Sources/SwabbleKit/WakeWordGate.swift +197 -0
- Swabble/Sources/swabble/CLI/CLIRegistry.swift +71 -0
- Swabble/Sources/swabble/Commands/DoctorCommand.swift +37 -0
- Swabble/Sources/swabble/Commands/HealthCommand.swift +16 -0
- Swabble/Sources/swabble/Commands/MicCommands.swift +62 -0
- Swabble/Sources/swabble/Commands/ServeCommand.swift +81 -0
- Swabble/Sources/swabble/Commands/ServiceCommands.swift +77 -0
- Swabble/Sources/swabble/Commands/SetupCommand.swift +26 -0
- Swabble/Sources/swabble/Commands/StartStopCommands.swift +35 -0
- Swabble/Sources/swabble/Commands/StatusCommand.swift +34 -0
- Swabble/Sources/swabble/Commands/TailLogCommand.swift +20 -0
- Swabble/Sources/swabble/Commands/TestHookCommand.swift +30 -0
- Swabble/Sources/swabble/Commands/TranscribeCommand.swift +61 -0
- Swabble/Sources/swabble/main.swift +151 -0
- Swabble/Tests/SwabbleKitTests/WakeWordGateTests.swift +63 -0
- Swabble/Tests/swabbleTests/ConfigTests.swift +23 -0
- Swabble/docs/spec.md +33 -0
- Swabble/scripts/format.sh +5 -0
- Swabble/scripts/lint.sh +9 -0
- apps/android/.gitignore +5 -0
- apps/android/README.md +51 -0
- apps/android/app/build.gradle.kts +128 -0
- apps/android/app/src/main/AndroidManifest.xml +49 -0
- apps/android/app/src/main/java/ai/openclaw/android/CameraHudState.kt +14 -0
- apps/android/app/src/main/java/ai/openclaw/android/DeviceNames.kt +26 -0
- apps/android/app/src/main/java/ai/openclaw/android/LocationMode.kt +15 -0
- apps/android/app/src/main/java/ai/openclaw/android/MainActivity.kt +130 -0
- apps/android/app/src/main/java/ai/openclaw/android/MainViewModel.kt +174 -0
- apps/android/app/src/main/java/ai/openclaw/android/NodeApp.kt +26 -0
- apps/android/app/src/main/java/ai/openclaw/android/NodeForegroundService.kt +180 -0
- apps/android/app/src/main/java/ai/openclaw/android/NodeRuntime.kt +1271 -0
- apps/android/app/src/main/java/ai/openclaw/android/PermissionRequester.kt +133 -0
.gitattributes
CHANGED
|
@@ -1,2 +1,7 @@
|
|
| 1 |
* text=auto eol=lf
|
| 2 |
README-header.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
* text=auto eol=lf
|
| 2 |
README-header.png filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-1024.png filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
apps/macos/Icon.icon/Assets/openclaw-mac.png filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
apps/macos/Sources/OpenClaw/Resources/OpenClaw.icns filter=lfs diff=lfs merge=lfs -text
|
Swabble/.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [main]
|
| 6 |
+
pull_request:
|
| 7 |
+
|
| 8 |
+
jobs:
|
| 9 |
+
build-and-test:
|
| 10 |
+
runs-on: macos-latest
|
| 11 |
+
defaults:
|
| 12 |
+
run:
|
| 13 |
+
shell: bash
|
| 14 |
+
working-directory: swabble
|
| 15 |
+
steps:
|
| 16 |
+
- name: Checkout swabble
|
| 17 |
+
uses: actions/checkout@v4
|
| 18 |
+
with:
|
| 19 |
+
path: swabble
|
| 20 |
+
|
| 21 |
+
- name: Select Xcode 26.1 (prefer 26.1.1)
|
| 22 |
+
run: |
|
| 23 |
+
set -euo pipefail
|
| 24 |
+
# pick the newest installed 26.1.x, fallback to newest 26.x
|
| 25 |
+
CANDIDATE="$(ls -d /Applications/Xcode_26.1*.app 2>/dev/null | sort -V | tail -1 || true)"
|
| 26 |
+
if [[ -z "$CANDIDATE" ]]; then
|
| 27 |
+
CANDIDATE="$(ls -d /Applications/Xcode_26*.app 2>/dev/null | sort -V | tail -1 || true)"
|
| 28 |
+
fi
|
| 29 |
+
if [[ -z "$CANDIDATE" ]]; then
|
| 30 |
+
echo "No Xcode 26.x found on runner" >&2
|
| 31 |
+
exit 1
|
| 32 |
+
fi
|
| 33 |
+
echo "Selecting $CANDIDATE"
|
| 34 |
+
sudo xcode-select -s "$CANDIDATE"
|
| 35 |
+
xcodebuild -version
|
| 36 |
+
|
| 37 |
+
- name: Show Swift version
|
| 38 |
+
run: swift --version
|
| 39 |
+
|
| 40 |
+
- name: Install tooling
|
| 41 |
+
run: |
|
| 42 |
+
brew update
|
| 43 |
+
brew install swiftlint swiftformat
|
| 44 |
+
|
| 45 |
+
- name: Format check
|
| 46 |
+
run: |
|
| 47 |
+
./scripts/format.sh
|
| 48 |
+
git diff --exit-code
|
| 49 |
+
|
| 50 |
+
- name: Lint
|
| 51 |
+
run: ./scripts/lint.sh
|
| 52 |
+
|
| 53 |
+
- name: Test
|
| 54 |
+
run: swift test --parallel
|
Swabble/.gitignore
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# macOS
|
| 2 |
+
.DS_Store
|
| 3 |
+
|
| 4 |
+
# SwiftPM / Build
|
| 5 |
+
/.build
|
| 6 |
+
/.swiftpm
|
| 7 |
+
/DerivedData
|
| 8 |
+
xcuserdata/
|
| 9 |
+
*.xcuserstate
|
| 10 |
+
|
| 11 |
+
# Editors
|
| 12 |
+
/.vscode
|
| 13 |
+
.idea/
|
| 14 |
+
|
| 15 |
+
# Xcode artifacts
|
| 16 |
+
*.hmap
|
| 17 |
+
*.ipa
|
| 18 |
+
*.dSYM.zip
|
| 19 |
+
*.dSYM
|
| 20 |
+
|
| 21 |
+
# Playgrounds
|
| 22 |
+
*.xcplayground
|
| 23 |
+
playground.xcworkspace
|
| 24 |
+
timeline.xctimeline
|
| 25 |
+
|
| 26 |
+
# Carthage
|
| 27 |
+
Carthage/Build/
|
| 28 |
+
|
| 29 |
+
# fastlane
|
| 30 |
+
fastlane/report.xml
|
| 31 |
+
fastlane/Preview.html
|
| 32 |
+
fastlane/screenshots/**/*.png
|
| 33 |
+
fastlane/test_output
|
Swabble/.swiftformat
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
--swiftversion 6.2
|
| 2 |
+
--indent 4
|
| 3 |
+
--maxwidth 120
|
| 4 |
+
--wraparguments before-first
|
| 5 |
+
--wrapcollections before-first
|
| 6 |
+
--stripunusedargs closure-only
|
| 7 |
+
--self remove
|
| 8 |
+
--header ""
|
Swabble/.swiftlint.yml
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SwiftLint for swabble
|
| 2 |
+
included:
|
| 3 |
+
- Sources
|
| 4 |
+
excluded:
|
| 5 |
+
- .build
|
| 6 |
+
- DerivedData
|
| 7 |
+
- "**/.swiftpm"
|
| 8 |
+
- "**/.build"
|
| 9 |
+
- "**/DerivedData"
|
| 10 |
+
- "**/.DS_Store"
|
| 11 |
+
opt_in_rules:
|
| 12 |
+
- array_init
|
| 13 |
+
- closure_spacing
|
| 14 |
+
- explicit_init
|
| 15 |
+
- fatal_error_message
|
| 16 |
+
- first_where
|
| 17 |
+
- joined_default_parameter
|
| 18 |
+
- last_where
|
| 19 |
+
- literal_expression_end_indentation
|
| 20 |
+
- multiline_arguments
|
| 21 |
+
- multiline_parameters
|
| 22 |
+
- operator_usage_whitespace
|
| 23 |
+
- redundant_nil_coalescing
|
| 24 |
+
- sorted_first_last
|
| 25 |
+
- switch_case_alignment
|
| 26 |
+
- vertical_parameter_alignment_on_call
|
| 27 |
+
- vertical_whitespace_opening_braces
|
| 28 |
+
- vertical_whitespace_closing_braces
|
| 29 |
+
|
| 30 |
+
disabled_rules:
|
| 31 |
+
- trailing_whitespace
|
| 32 |
+
- trailing_newline
|
| 33 |
+
- indentation_width
|
| 34 |
+
- identifier_name
|
| 35 |
+
- explicit_self
|
| 36 |
+
- file_header
|
| 37 |
+
- todo
|
| 38 |
+
|
| 39 |
+
line_length:
|
| 40 |
+
warning: 140
|
| 41 |
+
error: 180
|
| 42 |
+
|
| 43 |
+
reporter: "xcode"
|
Swabble/CHANGELOG.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Changelog
|
| 2 |
+
|
| 3 |
+
## 0.2.0 — 2025-12-23
|
| 4 |
+
|
| 5 |
+
### Highlights
|
| 6 |
+
- Added `SwabbleKit` (multi-platform wake-word gate utilities with segment-aware gap detection).
|
| 7 |
+
- Swabble package now supports iOS + macOS consumers; CLI remains macOS 26-only.
|
| 8 |
+
|
| 9 |
+
### Changes
|
| 10 |
+
- CLI wake-word matching/stripping routed through `SwabbleKit` helpers.
|
| 11 |
+
- Speech pipeline types now explicitly gated to macOS 26 / iOS 26 availability.
|
Swabble/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2025 Peter Steinberger
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
Swabble/Package.resolved
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"originHash" : "24a723309d7a0039d3df3051106f77ac1ed7068a02508e3a6804e41d757e6c72",
|
| 3 |
+
"pins" : [
|
| 4 |
+
{
|
| 5 |
+
"identity" : "commander",
|
| 6 |
+
"kind" : "remoteSourceControl",
|
| 7 |
+
"location" : "https://github.com/steipete/Commander.git",
|
| 8 |
+
"state" : {
|
| 9 |
+
"revision" : "9e349575c8e3c6745e81fe19e5bb5efa01b078ce",
|
| 10 |
+
"version" : "0.2.1"
|
| 11 |
+
}
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"identity" : "elevenlabskit",
|
| 15 |
+
"kind" : "remoteSourceControl",
|
| 16 |
+
"location" : "https://github.com/steipete/ElevenLabsKit",
|
| 17 |
+
"state" : {
|
| 18 |
+
"revision" : "7e3c948d8340abe3977014f3de020edf221e9269",
|
| 19 |
+
"version" : "0.1.0"
|
| 20 |
+
}
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"identity" : "swift-concurrency-extras",
|
| 24 |
+
"kind" : "remoteSourceControl",
|
| 25 |
+
"location" : "https://github.com/pointfreeco/swift-concurrency-extras",
|
| 26 |
+
"state" : {
|
| 27 |
+
"revision" : "5a3825302b1a0d744183200915a47b508c828e6f",
|
| 28 |
+
"version" : "1.3.2"
|
| 29 |
+
}
|
| 30 |
+
},
|
| 31 |
+
{
|
| 32 |
+
"identity" : "swift-syntax",
|
| 33 |
+
"kind" : "remoteSourceControl",
|
| 34 |
+
"location" : "https://github.com/swiftlang/swift-syntax.git",
|
| 35 |
+
"state" : {
|
| 36 |
+
"revision" : "0687f71944021d616d34d922343dcef086855920",
|
| 37 |
+
"version" : "600.0.1"
|
| 38 |
+
}
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"identity" : "swift-testing",
|
| 42 |
+
"kind" : "remoteSourceControl",
|
| 43 |
+
"location" : "https://github.com/apple/swift-testing",
|
| 44 |
+
"state" : {
|
| 45 |
+
"revision" : "399f76dcd91e4c688ca2301fa24a8cc6d9927211",
|
| 46 |
+
"version" : "0.99.0"
|
| 47 |
+
}
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"identity" : "swiftui-math",
|
| 51 |
+
"kind" : "remoteSourceControl",
|
| 52 |
+
"location" : "https://github.com/gonzalezreal/swiftui-math",
|
| 53 |
+
"state" : {
|
| 54 |
+
"revision" : "0b5c2cfaaec8d6193db206f675048eeb5ce95f71",
|
| 55 |
+
"version" : "0.1.0"
|
| 56 |
+
}
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"identity" : "textual",
|
| 60 |
+
"kind" : "remoteSourceControl",
|
| 61 |
+
"location" : "https://github.com/gonzalezreal/textual",
|
| 62 |
+
"state" : {
|
| 63 |
+
"revision" : "5b06b811c0f5313b6b84bbef98c635a630638c38",
|
| 64 |
+
"version" : "0.3.1"
|
| 65 |
+
}
|
| 66 |
+
}
|
| 67 |
+
],
|
| 68 |
+
"version" : 3
|
| 69 |
+
}
|
Swabble/Package.swift
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// swift-tools-version: 6.2
|
| 2 |
+
import PackageDescription
|
| 3 |
+
|
| 4 |
+
let package = Package(
|
| 5 |
+
name: "swabble",
|
| 6 |
+
platforms: [
|
| 7 |
+
.macOS(.v15),
|
| 8 |
+
.iOS(.v17),
|
| 9 |
+
],
|
| 10 |
+
products: [
|
| 11 |
+
.library(name: "Swabble", targets: ["Swabble"]),
|
| 12 |
+
.library(name: "SwabbleKit", targets: ["SwabbleKit"]),
|
| 13 |
+
.executable(name: "swabble", targets: ["SwabbleCLI"]),
|
| 14 |
+
],
|
| 15 |
+
dependencies: [
|
| 16 |
+
.package(url: "https://github.com/steipete/Commander.git", exact: "0.2.1"),
|
| 17 |
+
.package(url: "https://github.com/apple/swift-testing", from: "0.99.0"),
|
| 18 |
+
],
|
| 19 |
+
targets: [
|
| 20 |
+
.target(
|
| 21 |
+
name: "Swabble",
|
| 22 |
+
path: "Sources/SwabbleCore",
|
| 23 |
+
swiftSettings: []),
|
| 24 |
+
.target(
|
| 25 |
+
name: "SwabbleKit",
|
| 26 |
+
path: "Sources/SwabbleKit",
|
| 27 |
+
swiftSettings: [
|
| 28 |
+
.enableUpcomingFeature("StrictConcurrency"),
|
| 29 |
+
]),
|
| 30 |
+
.executableTarget(
|
| 31 |
+
name: "SwabbleCLI",
|
| 32 |
+
dependencies: [
|
| 33 |
+
"Swabble",
|
| 34 |
+
"SwabbleKit",
|
| 35 |
+
.product(name: "Commander", package: "Commander"),
|
| 36 |
+
],
|
| 37 |
+
path: "Sources/swabble"),
|
| 38 |
+
.testTarget(
|
| 39 |
+
name: "SwabbleKitTests",
|
| 40 |
+
dependencies: [
|
| 41 |
+
"SwabbleKit",
|
| 42 |
+
.product(name: "Testing", package: "swift-testing"),
|
| 43 |
+
],
|
| 44 |
+
swiftSettings: [
|
| 45 |
+
.enableUpcomingFeature("StrictConcurrency"),
|
| 46 |
+
.enableExperimentalFeature("SwiftTesting"),
|
| 47 |
+
]),
|
| 48 |
+
.testTarget(
|
| 49 |
+
name: "swabbleTests",
|
| 50 |
+
dependencies: [
|
| 51 |
+
"Swabble",
|
| 52 |
+
.product(name: "Testing", package: "swift-testing"),
|
| 53 |
+
]),
|
| 54 |
+
],
|
| 55 |
+
swiftLanguageModes: [.v6])
|
Swabble/README.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 🎙️ swabble — Speech.framework wake-word hook daemon (macOS 26)
|
| 2 |
+
|
| 3 |
+
swabble is a Swift 6.2 wake-word hook daemon. The CLI targets macOS 26 (SpeechAnalyzer + SpeechTranscriber). The shared `SwabbleKit` target is multi-platform and exposes wake-word gating utilities for iOS/macOS apps.
|
| 4 |
+
|
| 5 |
+
- **Local-only**: Speech.framework on-device models; zero network usage.
|
| 6 |
+
- **Wake word**: Default `clawd` (aliases `claude`), optional `--no-wake` bypass.
|
| 7 |
+
- **SwabbleKit**: Shared wake gate utilities (gap-based gating when you provide speech segments).
|
| 8 |
+
- **Hooks**: Run any command with prefix/env, cooldown, min_chars, timeout.
|
| 9 |
+
- **Services**: launchd helper stubs for start/stop/install.
|
| 10 |
+
- **File transcribe**: TXT or SRT with time ranges (using AttributedString splits).
|
| 11 |
+
|
| 12 |
+
## Quick start
|
| 13 |
+
```bash
|
| 14 |
+
# Install deps
|
| 15 |
+
brew install swiftformat swiftlint
|
| 16 |
+
|
| 17 |
+
# Build
|
| 18 |
+
swift build
|
| 19 |
+
|
| 20 |
+
# Write default config (~/.config/swabble/config.json)
|
| 21 |
+
swift run swabble setup
|
| 22 |
+
|
| 23 |
+
# Run foreground daemon
|
| 24 |
+
swift run swabble serve
|
| 25 |
+
|
| 26 |
+
# Test your hook
|
| 27 |
+
swift run swabble test-hook "hello world"
|
| 28 |
+
|
| 29 |
+
# Transcribe a file to SRT
|
| 30 |
+
swift run swabble transcribe /path/to/audio.m4a --format srt --output out.srt
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
## Use as a library
|
| 34 |
+
Add swabble as a SwiftPM dependency and import the `Swabble` or `SwabbleKit` product:
|
| 35 |
+
|
| 36 |
+
```swift
|
| 37 |
+
// Package.swift
|
| 38 |
+
dependencies: [
|
| 39 |
+
.package(url: "https://github.com/steipete/swabble.git", branch: "main"),
|
| 40 |
+
],
|
| 41 |
+
targets: [
|
| 42 |
+
.target(name: "MyApp", dependencies: [
|
| 43 |
+
.product(name: "Swabble", package: "swabble"), // Speech pipeline (macOS 26+ / iOS 26+)
|
| 44 |
+
.product(name: "SwabbleKit", package: "swabble"), // Wake-word gate utilities (iOS 17+ / macOS 15+)
|
| 45 |
+
]),
|
| 46 |
+
]
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
## CLI
|
| 50 |
+
- `serve` — foreground loop (mic → wake → hook)
|
| 51 |
+
- `transcribe <file>` — offline transcription (txt|srt)
|
| 52 |
+
- `test-hook "text"` — invoke configured hook
|
| 53 |
+
- `mic list|set <index>` — enumerate/select input device
|
| 54 |
+
- `setup` — write default config JSON
|
| 55 |
+
- `doctor` — check Speech auth & device availability
|
| 56 |
+
- `health` — prints `ok`
|
| 57 |
+
- `tail-log` — last 10 transcripts
|
| 58 |
+
- `status` — show wake state + recent transcripts
|
| 59 |
+
- `service install|uninstall|status` — user launchd plist (stub: prints launchctl commands)
|
| 60 |
+
- `start|stop|restart` — placeholders until full launchd wiring
|
| 61 |
+
|
| 62 |
+
All commands accept Commander runtime flags (`-v/--verbose`, `--json-output`, `--log-level`), plus `--config` where applicable.
|
| 63 |
+
|
| 64 |
+
## Config
|
| 65 |
+
`~/.config/swabble/config.json` (auto-created by `setup`):
|
| 66 |
+
```json
|
| 67 |
+
{
|
| 68 |
+
"audio": {"deviceName": "", "deviceIndex": -1, "sampleRate": 16000, "channels": 1},
|
| 69 |
+
"wake": {"enabled": true, "word": "clawd", "aliases": ["claude"]},
|
| 70 |
+
"hook": {
|
| 71 |
+
"command": "",
|
| 72 |
+
"args": [],
|
| 73 |
+
"prefix": "Voice swabble from ${hostname}: ",
|
| 74 |
+
"cooldownSeconds": 1,
|
| 75 |
+
"minCharacters": 24,
|
| 76 |
+
"timeoutSeconds": 5,
|
| 77 |
+
"env": {}
|
| 78 |
+
},
|
| 79 |
+
"logging": {"level": "info", "format": "text"},
|
| 80 |
+
"transcripts": {"enabled": true, "maxEntries": 50},
|
| 81 |
+
"speech": {"localeIdentifier": "en_US", "etiquetteReplacements": false}
|
| 82 |
+
}
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
- Config path override: `--config /path/to/config.json` on relevant commands.
|
| 86 |
+
- Transcripts persist to `~/Library/Application Support/swabble/transcripts.log`.
|
| 87 |
+
|
| 88 |
+
## Hook protocol
|
| 89 |
+
When a wake-gated transcript passes min_chars & cooldown, swabble runs:
|
| 90 |
+
```
|
| 91 |
+
<command> <args...> "<prefix><text>"
|
| 92 |
+
```
|
| 93 |
+
Environment variables:
|
| 94 |
+
- `SWABBLE_TEXT` — stripped transcript (wake word removed)
|
| 95 |
+
- `SWABBLE_PREFIX` — rendered prefix (hostname substituted)
|
| 96 |
+
- plus any `hook.env` key/values
|
| 97 |
+
|
| 98 |
+
## Speech pipeline
|
| 99 |
+
- `AVAudioEngine` tap → `BufferConverter` → `AnalyzerInput` → `SpeechAnalyzer` with a `SpeechTranscriber` module.
|
| 100 |
+
- Requests volatile + final results; the CLI uses text-only wake gating today.
|
| 101 |
+
- Authorization requested at first start; requires macOS 26 + new Speech.framework APIs.
|
| 102 |
+
|
| 103 |
+
## Development
|
| 104 |
+
- Format: `./scripts/format.sh` (uses local `.swiftformat`)
|
| 105 |
+
- Lint: `./scripts/lint.sh` (uses local `.swiftlint.yml`)
|
| 106 |
+
- Tests: `swift test` (uses swift-testing package)
|
| 107 |
+
|
| 108 |
+
## Roadmap
|
| 109 |
+
- launchd control (load/bootout, PID + status socket)
|
| 110 |
+
- JSON logging + PII redaction toggle
|
| 111 |
+
- Stronger wake-word detection and control socket status/health
|
Swabble/Sources/SwabbleCore/Config/Config.swift
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Foundation
|
| 2 |
+
|
| 3 |
+
public struct SwabbleConfig: Codable, Sendable {
|
| 4 |
+
public struct Audio: Codable, Sendable {
|
| 5 |
+
public var deviceName: String = ""
|
| 6 |
+
public var deviceIndex: Int = -1
|
| 7 |
+
public var sampleRate: Double = 16000
|
| 8 |
+
public var channels: Int = 1
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
public struct Wake: Codable, Sendable {
|
| 12 |
+
public var enabled: Bool = true
|
| 13 |
+
public var word: String = "clawd"
|
| 14 |
+
public var aliases: [String] = ["claude"]
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
public struct Hook: Codable, Sendable {
|
| 18 |
+
public var command: String = ""
|
| 19 |
+
public var args: [String] = []
|
| 20 |
+
public var prefix: String = "Voice swabble from ${hostname}: "
|
| 21 |
+
public var cooldownSeconds: Double = 1
|
| 22 |
+
public var minCharacters: Int = 24
|
| 23 |
+
public var timeoutSeconds: Double = 5
|
| 24 |
+
public var env: [String: String] = [:]
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
public struct Logging: Codable, Sendable {
|
| 28 |
+
public var level: String = "info"
|
| 29 |
+
public var format: String = "text" // text|json placeholder
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
public struct Transcripts: Codable, Sendable {
|
| 33 |
+
public var enabled: Bool = true
|
| 34 |
+
public var maxEntries: Int = 50
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
public struct Speech: Codable, Sendable {
|
| 38 |
+
public var localeIdentifier: String = Locale.current.identifier
|
| 39 |
+
public var etiquetteReplacements: Bool = false
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
public var audio = Audio()
|
| 43 |
+
public var wake = Wake()
|
| 44 |
+
public var hook = Hook()
|
| 45 |
+
public var logging = Logging()
|
| 46 |
+
public var transcripts = Transcripts()
|
| 47 |
+
public var speech = Speech()
|
| 48 |
+
|
| 49 |
+
public static let defaultPath = FileManager.default
|
| 50 |
+
.homeDirectoryForCurrentUser
|
| 51 |
+
.appendingPathComponent(".config/swabble/config.json")
|
| 52 |
+
|
| 53 |
+
public init() {}
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
public enum ConfigError: Error {
|
| 57 |
+
case missingConfig
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
public enum ConfigLoader {
|
| 61 |
+
public static func load(at path: URL?) throws -> SwabbleConfig {
|
| 62 |
+
let url = path ?? SwabbleConfig.defaultPath
|
| 63 |
+
if !FileManager.default.fileExists(atPath: url.path) {
|
| 64 |
+
throw ConfigError.missingConfig
|
| 65 |
+
}
|
| 66 |
+
let data = try Data(contentsOf: url)
|
| 67 |
+
return try JSONDecoder().decode(SwabbleConfig.self, from: data)
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
public static func save(_ config: SwabbleConfig, at path: URL?) throws {
|
| 71 |
+
let url = path ?? SwabbleConfig.defaultPath
|
| 72 |
+
let dir = url.deletingLastPathComponent()
|
| 73 |
+
try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
| 74 |
+
let data = try JSONEncoder().encode(config)
|
| 75 |
+
try data.write(to: url)
|
| 76 |
+
}
|
| 77 |
+
}
|
Swabble/Sources/SwabbleCore/Hooks/HookExecutor.swift
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Foundation
|
| 2 |
+
|
| 3 |
+
public struct HookJob: Sendable {
|
| 4 |
+
public let text: String
|
| 5 |
+
public let timestamp: Date
|
| 6 |
+
|
| 7 |
+
public init(text: String, timestamp: Date) {
|
| 8 |
+
self.text = text
|
| 9 |
+
self.timestamp = timestamp
|
| 10 |
+
}
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
public actor HookExecutor {
|
| 14 |
+
private let config: SwabbleConfig
|
| 15 |
+
private var lastRun: Date?
|
| 16 |
+
private let hostname: String
|
| 17 |
+
|
| 18 |
+
public init(config: SwabbleConfig) {
|
| 19 |
+
self.config = config
|
| 20 |
+
hostname = Host.current().localizedName ?? "host"
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
public func shouldRun() -> Bool {
|
| 24 |
+
guard config.hook.cooldownSeconds > 0 else { return true }
|
| 25 |
+
if let lastRun, Date().timeIntervalSince(lastRun) < config.hook.cooldownSeconds {
|
| 26 |
+
return false
|
| 27 |
+
}
|
| 28 |
+
return true
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
public func run(job: HookJob) async throws {
|
| 32 |
+
guard shouldRun() else { return }
|
| 33 |
+
guard !config.hook.command.isEmpty else { throw NSError(
|
| 34 |
+
domain: "Hook",
|
| 35 |
+
code: 1,
|
| 36 |
+
userInfo: [NSLocalizedDescriptionKey: "hook command not set"]) }
|
| 37 |
+
|
| 38 |
+
let prefix = config.hook.prefix.replacingOccurrences(of: "${hostname}", with: hostname)
|
| 39 |
+
let payload = prefix + job.text
|
| 40 |
+
|
| 41 |
+
let process = Process()
|
| 42 |
+
process.executableURL = URL(fileURLWithPath: config.hook.command)
|
| 43 |
+
process.arguments = config.hook.args + [payload]
|
| 44 |
+
|
| 45 |
+
var env = ProcessInfo.processInfo.environment
|
| 46 |
+
env["SWABBLE_TEXT"] = job.text
|
| 47 |
+
env["SWABBLE_PREFIX"] = prefix
|
| 48 |
+
for (k, v) in config.hook.env {
|
| 49 |
+
env[k] = v
|
| 50 |
+
}
|
| 51 |
+
process.environment = env
|
| 52 |
+
|
| 53 |
+
let pipe = Pipe()
|
| 54 |
+
process.standardOutput = pipe
|
| 55 |
+
process.standardError = pipe
|
| 56 |
+
|
| 57 |
+
try process.run()
|
| 58 |
+
|
| 59 |
+
let timeoutNanos = UInt64(max(config.hook.timeoutSeconds, 0.1) * 1_000_000_000)
|
| 60 |
+
try await withThrowingTaskGroup(of: Void.self) { group in
|
| 61 |
+
group.addTask {
|
| 62 |
+
process.waitUntilExit()
|
| 63 |
+
}
|
| 64 |
+
group.addTask {
|
| 65 |
+
try await Task.sleep(nanoseconds: timeoutNanos)
|
| 66 |
+
if process.isRunning {
|
| 67 |
+
process.terminate()
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
try await group.next()
|
| 71 |
+
group.cancelAll()
|
| 72 |
+
}
|
| 73 |
+
lastRun = Date()
|
| 74 |
+
}
|
| 75 |
+
}
|
Swabble/Sources/SwabbleCore/Speech/BufferConverter.swift
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
@preconcurrency import AVFoundation
|
| 2 |
+
import Foundation
|
| 3 |
+
|
| 4 |
+
final class BufferConverter {
|
| 5 |
+
private final class Box<T>: @unchecked Sendable { var value: T; init(_ value: T) { self.value = value } }
|
| 6 |
+
enum ConverterError: Swift.Error {
|
| 7 |
+
case failedToCreateConverter
|
| 8 |
+
case failedToCreateConversionBuffer
|
| 9 |
+
case conversionFailed(NSError?)
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
private var converter: AVAudioConverter?
|
| 13 |
+
|
| 14 |
+
func convert(_ buffer: AVAudioPCMBuffer, to format: AVAudioFormat) throws -> AVAudioPCMBuffer {
|
| 15 |
+
let inputFormat = buffer.format
|
| 16 |
+
if inputFormat == format {
|
| 17 |
+
return buffer
|
| 18 |
+
}
|
| 19 |
+
if converter == nil || converter?.outputFormat != format {
|
| 20 |
+
converter = AVAudioConverter(from: inputFormat, to: format)
|
| 21 |
+
converter?.primeMethod = .none
|
| 22 |
+
}
|
| 23 |
+
guard let converter else { throw ConverterError.failedToCreateConverter }
|
| 24 |
+
|
| 25 |
+
let sampleRateRatio = converter.outputFormat.sampleRate / converter.inputFormat.sampleRate
|
| 26 |
+
let scaledInputFrameLength = Double(buffer.frameLength) * sampleRateRatio
|
| 27 |
+
let frameCapacity = AVAudioFrameCount(scaledInputFrameLength.rounded(.up))
|
| 28 |
+
guard let conversionBuffer = AVAudioPCMBuffer(pcmFormat: converter.outputFormat, frameCapacity: frameCapacity)
|
| 29 |
+
else {
|
| 30 |
+
throw ConverterError.failedToCreateConversionBuffer
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
var nsError: NSError?
|
| 34 |
+
let consumed = Box(false)
|
| 35 |
+
let inputBuffer = buffer
|
| 36 |
+
let status = converter.convert(to: conversionBuffer, error: &nsError) { _, statusPtr in
|
| 37 |
+
if consumed.value {
|
| 38 |
+
statusPtr.pointee = .noDataNow
|
| 39 |
+
return nil
|
| 40 |
+
}
|
| 41 |
+
consumed.value = true
|
| 42 |
+
statusPtr.pointee = .haveData
|
| 43 |
+
return inputBuffer
|
| 44 |
+
}
|
| 45 |
+
if status == .error {
|
| 46 |
+
throw ConverterError.conversionFailed(nsError)
|
| 47 |
+
}
|
| 48 |
+
return conversionBuffer
|
| 49 |
+
}
|
| 50 |
+
}
|
Swabble/Sources/SwabbleCore/Speech/SpeechPipeline.swift
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import AVFoundation
|
| 2 |
+
import Foundation
|
| 3 |
+
import Speech
|
| 4 |
+
|
| 5 |
+
@available(macOS 26.0, iOS 26.0, *)
|
| 6 |
+
public struct SpeechSegment: Sendable {
|
| 7 |
+
public let text: String
|
| 8 |
+
public let isFinal: Bool
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
@available(macOS 26.0, iOS 26.0, *)
|
| 12 |
+
public enum SpeechPipelineError: Error {
|
| 13 |
+
case authorizationDenied
|
| 14 |
+
case analyzerFormatUnavailable
|
| 15 |
+
case transcriberUnavailable
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
/// Live microphone → SpeechAnalyzer → SpeechTranscriber pipeline.
|
| 19 |
+
@available(macOS 26.0, iOS 26.0, *)
|
| 20 |
+
public actor SpeechPipeline {
|
| 21 |
+
private struct UnsafeBuffer: @unchecked Sendable { let buffer: AVAudioPCMBuffer }
|
| 22 |
+
|
| 23 |
+
private var engine = AVAudioEngine()
|
| 24 |
+
private var transcriber: SpeechTranscriber?
|
| 25 |
+
private var analyzer: SpeechAnalyzer?
|
| 26 |
+
private var inputContinuation: AsyncStream<AnalyzerInput>.Continuation?
|
| 27 |
+
private var resultTask: Task<Void, Never>?
|
| 28 |
+
private let converter = BufferConverter()
|
| 29 |
+
|
| 30 |
+
public init() {}
|
| 31 |
+
|
| 32 |
+
public func start(localeIdentifier: String, etiquette: Bool) async throws -> AsyncStream<SpeechSegment> {
|
| 33 |
+
let auth = await requestAuthorizationIfNeeded()
|
| 34 |
+
guard auth == .authorized else { throw SpeechPipelineError.authorizationDenied }
|
| 35 |
+
|
| 36 |
+
let transcriberModule = SpeechTranscriber(
|
| 37 |
+
locale: Locale(identifier: localeIdentifier),
|
| 38 |
+
transcriptionOptions: etiquette ? [.etiquetteReplacements] : [],
|
| 39 |
+
reportingOptions: [.volatileResults],
|
| 40 |
+
attributeOptions: [])
|
| 41 |
+
transcriber = transcriberModule
|
| 42 |
+
|
| 43 |
+
guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriberModule])
|
| 44 |
+
else {
|
| 45 |
+
throw SpeechPipelineError.analyzerFormatUnavailable
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
analyzer = SpeechAnalyzer(modules: [transcriberModule])
|
| 49 |
+
let (stream, continuation) = AsyncStream<AnalyzerInput>.makeStream()
|
| 50 |
+
inputContinuation = continuation
|
| 51 |
+
|
| 52 |
+
let inputNode = engine.inputNode
|
| 53 |
+
let inputFormat = inputNode.outputFormat(forBus: 0)
|
| 54 |
+
inputNode.removeTap(onBus: 0)
|
| 55 |
+
inputNode.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { [weak self] buffer, _ in
|
| 56 |
+
guard let self else { return }
|
| 57 |
+
let boxed = UnsafeBuffer(buffer: buffer)
|
| 58 |
+
Task { await self.handleBuffer(boxed.buffer, targetFormat: analyzerFormat) }
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
engine.prepare()
|
| 62 |
+
try engine.start()
|
| 63 |
+
try await analyzer?.start(inputSequence: stream)
|
| 64 |
+
|
| 65 |
+
guard let transcriberForStream = transcriber else {
|
| 66 |
+
throw SpeechPipelineError.transcriberUnavailable
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
return AsyncStream { continuation in
|
| 70 |
+
self.resultTask = Task {
|
| 71 |
+
do {
|
| 72 |
+
for try await result in transcriberForStream.results {
|
| 73 |
+
let seg = SpeechSegment(text: String(result.text.characters), isFinal: result.isFinal)
|
| 74 |
+
continuation.yield(seg)
|
| 75 |
+
}
|
| 76 |
+
} catch {
|
| 77 |
+
// swallow errors and finish
|
| 78 |
+
}
|
| 79 |
+
continuation.finish()
|
| 80 |
+
}
|
| 81 |
+
continuation.onTermination = { _ in
|
| 82 |
+
Task { await self.stop() }
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
public func stop() async {
|
| 88 |
+
resultTask?.cancel()
|
| 89 |
+
inputContinuation?.finish()
|
| 90 |
+
engine.inputNode.removeTap(onBus: 0)
|
| 91 |
+
engine.stop()
|
| 92 |
+
try? await analyzer?.finalizeAndFinishThroughEndOfInput()
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
private func handleBuffer(_ buffer: AVAudioPCMBuffer, targetFormat: AVAudioFormat) async {
|
| 96 |
+
do {
|
| 97 |
+
let converted = try converter.convert(buffer, to: targetFormat)
|
| 98 |
+
let input = AnalyzerInput(buffer: converted)
|
| 99 |
+
inputContinuation?.yield(input)
|
| 100 |
+
} catch {
|
| 101 |
+
// drop on conversion failure
|
| 102 |
+
}
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
private func requestAuthorizationIfNeeded() async -> SFSpeechRecognizerAuthorizationStatus {
|
| 106 |
+
let current = SFSpeechRecognizer.authorizationStatus()
|
| 107 |
+
guard current == .notDetermined else { return current }
|
| 108 |
+
return await withCheckedContinuation { continuation in
|
| 109 |
+
SFSpeechRecognizer.requestAuthorization { status in
|
| 110 |
+
continuation.resume(returning: status)
|
| 111 |
+
}
|
| 112 |
+
}
|
| 113 |
+
}
|
| 114 |
+
}
|
Swabble/Sources/SwabbleCore/Support/AttributedString+Sentences.swift
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import CoreMedia
|
| 2 |
+
import Foundation
|
| 3 |
+
import NaturalLanguage
|
| 4 |
+
|
| 5 |
+
extension AttributedString {
|
| 6 |
+
public func sentences(maxLength: Int? = nil) -> [AttributedString] {
|
| 7 |
+
let tokenizer = NLTokenizer(unit: .sentence)
|
| 8 |
+
let string = String(characters)
|
| 9 |
+
tokenizer.string = string
|
| 10 |
+
let sentenceRanges = tokenizer.tokens(for: string.startIndex..<string.endIndex).map {
|
| 11 |
+
(
|
| 12 |
+
$0,
|
| 13 |
+
AttributedString.Index($0.lowerBound, within: self)!
|
| 14 |
+
..<
|
| 15 |
+
AttributedString.Index($0.upperBound, within: self)!)
|
| 16 |
+
}
|
| 17 |
+
let ranges = sentenceRanges.flatMap { sentenceStringRange, sentenceRange in
|
| 18 |
+
let sentence = self[sentenceRange]
|
| 19 |
+
guard let maxLength, sentence.characters.count > maxLength else {
|
| 20 |
+
return [sentenceRange]
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
let wordTokenizer = NLTokenizer(unit: .word)
|
| 24 |
+
wordTokenizer.string = string
|
| 25 |
+
var wordRanges = wordTokenizer.tokens(for: sentenceStringRange).map {
|
| 26 |
+
AttributedString.Index($0.lowerBound, within: self)!
|
| 27 |
+
..<
|
| 28 |
+
AttributedString.Index($0.upperBound, within: self)!
|
| 29 |
+
}
|
| 30 |
+
guard !wordRanges.isEmpty else { return [sentenceRange] }
|
| 31 |
+
wordRanges[0] = sentenceRange.lowerBound..<wordRanges[0].upperBound
|
| 32 |
+
wordRanges[wordRanges.count - 1] = wordRanges[wordRanges.count - 1].lowerBound..<sentenceRange.upperBound
|
| 33 |
+
|
| 34 |
+
var ranges: [Range<AttributedString.Index>] = []
|
| 35 |
+
for wordRange in wordRanges {
|
| 36 |
+
if let lastRange = ranges.last,
|
| 37 |
+
self[lastRange].characters.count + self[wordRange].characters.count <= maxLength {
|
| 38 |
+
ranges[ranges.count - 1] = lastRange.lowerBound..<wordRange.upperBound
|
| 39 |
+
} else {
|
| 40 |
+
ranges.append(wordRange)
|
| 41 |
+
}
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
return ranges
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
return ranges.compactMap { range in
|
| 48 |
+
let audioTimeRanges = self[range].runs.filter {
|
| 49 |
+
!String(self[$0.range].characters)
|
| 50 |
+
.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
| 51 |
+
}.compactMap(\.audioTimeRange)
|
| 52 |
+
guard !audioTimeRanges.isEmpty else { return nil }
|
| 53 |
+
let start = audioTimeRanges.first!.start
|
| 54 |
+
let end = audioTimeRanges.last!.end
|
| 55 |
+
var attributes = AttributeContainer()
|
| 56 |
+
attributes[AttributeScopes.SpeechAttributes.TimeRangeAttribute.self] = CMTimeRange(
|
| 57 |
+
start: start,
|
| 58 |
+
end: end)
|
| 59 |
+
return AttributedString(self[range].characters, attributes: attributes)
|
| 60 |
+
}
|
| 61 |
+
}
|
| 62 |
+
}
|
Swabble/Sources/SwabbleCore/Support/Logging.swift
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Foundation
|
| 2 |
+
|
| 3 |
+
public enum LogLevel: String, Comparable, CaseIterable, Sendable {
|
| 4 |
+
case trace, debug, info, warn, error
|
| 5 |
+
|
| 6 |
+
var rank: Int {
|
| 7 |
+
switch self {
|
| 8 |
+
case .trace: 0
|
| 9 |
+
case .debug: 1
|
| 10 |
+
case .info: 2
|
| 11 |
+
case .warn: 3
|
| 12 |
+
case .error: 4
|
| 13 |
+
}
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
public static func < (lhs: LogLevel, rhs: LogLevel) -> Bool { lhs.rank < rhs.rank }
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
public struct Logger: Sendable {
|
| 20 |
+
public let level: LogLevel
|
| 21 |
+
|
| 22 |
+
public init(level: LogLevel) { self.level = level }
|
| 23 |
+
|
| 24 |
+
public func log(_ level: LogLevel, _ message: String) {
|
| 25 |
+
guard level >= self.level else { return }
|
| 26 |
+
let ts = ISO8601DateFormatter().string(from: Date())
|
| 27 |
+
print("[\(level.rawValue.uppercased())] \(ts) | \(message)")
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
public func trace(_ msg: String) { log(.trace, msg) }
|
| 31 |
+
public func debug(_ msg: String) { log(.debug, msg) }
|
| 32 |
+
public func info(_ msg: String) { log(.info, msg) }
|
| 33 |
+
public func warn(_ msg: String) { log(.warn, msg) }
|
| 34 |
+
public func error(_ msg: String) { log(.error, msg) }
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
extension LogLevel {
|
| 38 |
+
public init?(configValue: String) {
|
| 39 |
+
self.init(rawValue: configValue.lowercased())
|
| 40 |
+
}
|
| 41 |
+
}
|
Swabble/Sources/SwabbleCore/Support/OutputFormat.swift
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import CoreMedia
|
| 2 |
+
import Foundation
|
| 3 |
+
|
| 4 |
+
public enum OutputFormat: String {
|
| 5 |
+
case txt
|
| 6 |
+
case srt
|
| 7 |
+
|
| 8 |
+
public var needsAudioTimeRange: Bool {
|
| 9 |
+
switch self {
|
| 10 |
+
case .srt: true
|
| 11 |
+
default: false
|
| 12 |
+
}
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
public func text(for transcript: AttributedString, maxLength: Int) -> String {
|
| 16 |
+
switch self {
|
| 17 |
+
case .txt:
|
| 18 |
+
return String(transcript.characters)
|
| 19 |
+
case .srt:
|
| 20 |
+
func format(_ timeInterval: TimeInterval) -> String {
|
| 21 |
+
let ms = Int(timeInterval.truncatingRemainder(dividingBy: 1) * 1000)
|
| 22 |
+
let s = Int(timeInterval) % 60
|
| 23 |
+
let m = (Int(timeInterval) / 60) % 60
|
| 24 |
+
let h = Int(timeInterval) / 60 / 60
|
| 25 |
+
return String(format: "%0.2d:%0.2d:%0.2d,%0.3d", h, m, s, ms)
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
return transcript.sentences(maxLength: maxLength).compactMap { (sentence: AttributedString) -> (
|
| 29 |
+
CMTimeRange,
|
| 30 |
+
String)? in
|
| 31 |
+
guard let timeRange = sentence.audioTimeRange else { return nil }
|
| 32 |
+
return (timeRange, String(sentence.characters))
|
| 33 |
+
}.enumerated().map { index, run in
|
| 34 |
+
let (timeRange, text) = run
|
| 35 |
+
return """
|
| 36 |
+
|
| 37 |
+
\(index + 1)
|
| 38 |
+
\(format(timeRange.start.seconds)) --> \(format(timeRange.end.seconds))
|
| 39 |
+
\(text.trimmingCharacters(in: .whitespacesAndNewlines))
|
| 40 |
+
|
| 41 |
+
"""
|
| 42 |
+
}.joined().trimmingCharacters(in: .whitespacesAndNewlines)
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
}
|
Swabble/Sources/SwabbleCore/Support/TranscriptsStore.swift
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Foundation
|
| 2 |
+
|
| 3 |
+
public actor TranscriptsStore {
|
| 4 |
+
public static let shared = TranscriptsStore()
|
| 5 |
+
|
| 6 |
+
private var entries: [String] = []
|
| 7 |
+
private let limit = 100
|
| 8 |
+
private let fileURL: URL
|
| 9 |
+
|
| 10 |
+
public init() {
|
| 11 |
+
let dir = FileManager.default.homeDirectoryForCurrentUser
|
| 12 |
+
.appendingPathComponent("Library/Application Support/swabble", isDirectory: true)
|
| 13 |
+
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
| 14 |
+
fileURL = dir.appendingPathComponent("transcripts.log")
|
| 15 |
+
if let data = try? Data(contentsOf: fileURL),
|
| 16 |
+
let text = String(data: data, encoding: .utf8) {
|
| 17 |
+
entries = text.split(separator: "\n").map(String.init).suffix(limit)
|
| 18 |
+
}
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
public func append(text: String) {
|
| 22 |
+
entries.append(text)
|
| 23 |
+
if entries.count > limit {
|
| 24 |
+
entries.removeFirst(entries.count - limit)
|
| 25 |
+
}
|
| 26 |
+
let body = entries.joined(separator: "\n")
|
| 27 |
+
try? body.write(to: fileURL, atomically: false, encoding: .utf8)
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
public func latest() -> [String] { entries }
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
extension String {
|
| 34 |
+
private func appendLine(to url: URL) throws {
|
| 35 |
+
let data = (self + "\n").data(using: .utf8) ?? Data()
|
| 36 |
+
if FileManager.default.fileExists(atPath: url.path) {
|
| 37 |
+
let handle = try FileHandle(forWritingTo: url)
|
| 38 |
+
try handle.seekToEnd()
|
| 39 |
+
try handle.write(contentsOf: data)
|
| 40 |
+
try handle.close()
|
| 41 |
+
} else {
|
| 42 |
+
try data.write(to: url)
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
}
|
Swabble/Sources/SwabbleKit/WakeWordGate.swift
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Foundation
|
| 2 |
+
|
| 3 |
+
public struct WakeWordSegment: Sendable, Equatable {
|
| 4 |
+
public let text: String
|
| 5 |
+
public let start: TimeInterval
|
| 6 |
+
public let duration: TimeInterval
|
| 7 |
+
public let range: Range<String.Index>?
|
| 8 |
+
|
| 9 |
+
public init(text: String, start: TimeInterval, duration: TimeInterval, range: Range<String.Index>? = nil) {
|
| 10 |
+
self.text = text
|
| 11 |
+
self.start = start
|
| 12 |
+
self.duration = duration
|
| 13 |
+
self.range = range
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
public var end: TimeInterval { start + duration }
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
public struct WakeWordGateConfig: Sendable, Equatable {
|
| 20 |
+
public var triggers: [String]
|
| 21 |
+
public var minPostTriggerGap: TimeInterval
|
| 22 |
+
public var minCommandLength: Int
|
| 23 |
+
|
| 24 |
+
public init(
|
| 25 |
+
triggers: [String],
|
| 26 |
+
minPostTriggerGap: TimeInterval = 0.45,
|
| 27 |
+
minCommandLength: Int = 1) {
|
| 28 |
+
self.triggers = triggers
|
| 29 |
+
self.minPostTriggerGap = minPostTriggerGap
|
| 30 |
+
self.minCommandLength = minCommandLength
|
| 31 |
+
}
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
public struct WakeWordGateMatch: Sendable, Equatable {
|
| 35 |
+
public let triggerEndTime: TimeInterval
|
| 36 |
+
public let postGap: TimeInterval
|
| 37 |
+
public let command: String
|
| 38 |
+
|
| 39 |
+
public init(triggerEndTime: TimeInterval, postGap: TimeInterval, command: String) {
|
| 40 |
+
self.triggerEndTime = triggerEndTime
|
| 41 |
+
self.postGap = postGap
|
| 42 |
+
self.command = command
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
public enum WakeWordGate {
|
| 47 |
+
private struct Token {
|
| 48 |
+
let normalized: String
|
| 49 |
+
let start: TimeInterval
|
| 50 |
+
let end: TimeInterval
|
| 51 |
+
let range: Range<String.Index>?
|
| 52 |
+
let text: String
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
private struct TriggerTokens {
|
| 56 |
+
let tokens: [String]
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
private struct MatchCandidate {
|
| 60 |
+
let index: Int
|
| 61 |
+
let triggerEnd: TimeInterval
|
| 62 |
+
let gap: TimeInterval
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
public static func match(
|
| 66 |
+
transcript: String,
|
| 67 |
+
segments: [WakeWordSegment],
|
| 68 |
+
config: WakeWordGateConfig)
|
| 69 |
+
-> WakeWordGateMatch? {
|
| 70 |
+
let triggerTokens = normalizeTriggers(config.triggers)
|
| 71 |
+
guard !triggerTokens.isEmpty else { return nil }
|
| 72 |
+
|
| 73 |
+
let tokens = normalizeSegments(segments)
|
| 74 |
+
guard !tokens.isEmpty else { return nil }
|
| 75 |
+
|
| 76 |
+
var best: MatchCandidate?
|
| 77 |
+
|
| 78 |
+
for trigger in triggerTokens {
|
| 79 |
+
let count = trigger.tokens.count
|
| 80 |
+
guard count > 0, tokens.count > count else { continue }
|
| 81 |
+
for i in 0...(tokens.count - count - 1) {
|
| 82 |
+
let matched = (0..<count).allSatisfy { tokens[i + $0].normalized == trigger.tokens[$0] }
|
| 83 |
+
if !matched { continue }
|
| 84 |
+
|
| 85 |
+
let triggerEnd = tokens[i + count - 1].end
|
| 86 |
+
let nextToken = tokens[i + count]
|
| 87 |
+
let gap = nextToken.start - triggerEnd
|
| 88 |
+
if gap < config.minPostTriggerGap { continue }
|
| 89 |
+
|
| 90 |
+
if let best, i <= best.index { continue }
|
| 91 |
+
|
| 92 |
+
best = MatchCandidate(index: i, triggerEnd: triggerEnd, gap: gap)
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
guard let best else { return nil }
|
| 97 |
+
let command = commandText(transcript: transcript, segments: segments, triggerEndTime: best.triggerEnd)
|
| 98 |
+
.trimmingCharacters(in: Self.whitespaceAndPunctuation)
|
| 99 |
+
guard command.count >= config.minCommandLength else { return nil }
|
| 100 |
+
return WakeWordGateMatch(triggerEndTime: best.triggerEnd, postGap: best.gap, command: command)
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
public static func commandText(
|
| 104 |
+
transcript: String,
|
| 105 |
+
segments: [WakeWordSegment],
|
| 106 |
+
triggerEndTime: TimeInterval)
|
| 107 |
+
-> String {
|
| 108 |
+
let threshold = triggerEndTime + 0.001
|
| 109 |
+
for segment in segments where segment.start >= threshold {
|
| 110 |
+
if normalizeToken(segment.text).isEmpty { continue }
|
| 111 |
+
if let range = segment.range {
|
| 112 |
+
let slice = transcript[range.lowerBound...]
|
| 113 |
+
return String(slice).trimmingCharacters(in: Self.whitespaceAndPunctuation)
|
| 114 |
+
}
|
| 115 |
+
break
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
let text = segments
|
| 119 |
+
.filter { $0.start >= threshold && !normalizeToken($0.text).isEmpty }
|
| 120 |
+
.map(\.text)
|
| 121 |
+
.joined(separator: " ")
|
| 122 |
+
return text.trimmingCharacters(in: Self.whitespaceAndPunctuation)
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
public static func matchesTextOnly(text: String, triggers: [String]) -> Bool {
|
| 126 |
+
guard !text.isEmpty else { return false }
|
| 127 |
+
let normalized = text.lowercased()
|
| 128 |
+
for trigger in triggers {
|
| 129 |
+
let token = trigger.trimmingCharacters(in: whitespaceAndPunctuation).lowercased()
|
| 130 |
+
if token.isEmpty { continue }
|
| 131 |
+
if normalized.contains(token) { return true }
|
| 132 |
+
}
|
| 133 |
+
return false
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
public static func stripWake(text: String, triggers: [String]) -> String {
|
| 137 |
+
var out = text
|
| 138 |
+
for trigger in triggers {
|
| 139 |
+
let token = trigger.trimmingCharacters(in: whitespaceAndPunctuation)
|
| 140 |
+
guard !token.isEmpty else { continue }
|
| 141 |
+
out = out.replacingOccurrences(of: token, with: "", options: [.caseInsensitive])
|
| 142 |
+
}
|
| 143 |
+
return out.trimmingCharacters(in: whitespaceAndPunctuation)
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
private static func normalizeTriggers(_ triggers: [String]) -> [TriggerTokens] {
|
| 147 |
+
var output: [TriggerTokens] = []
|
| 148 |
+
for trigger in triggers {
|
| 149 |
+
let tokens = trigger
|
| 150 |
+
.split(whereSeparator: { $0.isWhitespace })
|
| 151 |
+
.map { normalizeToken(String($0)) }
|
| 152 |
+
.filter { !$0.isEmpty }
|
| 153 |
+
if tokens.isEmpty { continue }
|
| 154 |
+
output.append(TriggerTokens(tokens: tokens))
|
| 155 |
+
}
|
| 156 |
+
return output
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
private static func normalizeSegments(_ segments: [WakeWordSegment]) -> [Token] {
|
| 160 |
+
segments.compactMap { segment in
|
| 161 |
+
let normalized = normalizeToken(segment.text)
|
| 162 |
+
guard !normalized.isEmpty else { return nil }
|
| 163 |
+
return Token(
|
| 164 |
+
normalized: normalized,
|
| 165 |
+
start: segment.start,
|
| 166 |
+
end: segment.end,
|
| 167 |
+
range: segment.range,
|
| 168 |
+
text: segment.text)
|
| 169 |
+
}
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
private static func normalizeToken(_ token: String) -> String {
|
| 173 |
+
token
|
| 174 |
+
.trimmingCharacters(in: whitespaceAndPunctuation)
|
| 175 |
+
.lowercased()
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
private static let whitespaceAndPunctuation = CharacterSet.whitespacesAndNewlines
|
| 179 |
+
.union(.punctuationCharacters)
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
#if canImport(Speech)
|
| 183 |
+
import Speech
|
| 184 |
+
|
| 185 |
+
public enum WakeWordSpeechSegments {
|
| 186 |
+
public static func from(transcription: SFTranscription, transcript: String) -> [WakeWordSegment] {
|
| 187 |
+
transcription.segments.map { segment in
|
| 188 |
+
let range = Range(segment.substringRange, in: transcript)
|
| 189 |
+
return WakeWordSegment(
|
| 190 |
+
text: segment.substring,
|
| 191 |
+
start: segment.timestamp,
|
| 192 |
+
duration: segment.duration,
|
| 193 |
+
range: range)
|
| 194 |
+
}
|
| 195 |
+
}
|
| 196 |
+
}
|
| 197 |
+
#endif
|
Swabble/Sources/swabble/CLI/CLIRegistry.swift
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
|
| 4 |
+
@available(macOS 26.0, *)
|
| 5 |
+
@MainActor
|
| 6 |
+
enum CLIRegistry {
|
| 7 |
+
static var descriptors: [CommandDescriptor] {
|
| 8 |
+
let serveDesc = descriptor(for: ServeCommand.self)
|
| 9 |
+
let transcribeDesc = descriptor(for: TranscribeCommand.self)
|
| 10 |
+
let testHookDesc = descriptor(for: TestHookCommand.self)
|
| 11 |
+
let micList = descriptor(for: MicList.self)
|
| 12 |
+
let micSet = descriptor(for: MicSet.self)
|
| 13 |
+
let micRoot = CommandDescriptor(
|
| 14 |
+
name: "mic",
|
| 15 |
+
abstract: "Microphone management",
|
| 16 |
+
discussion: nil,
|
| 17 |
+
signature: CommandSignature(),
|
| 18 |
+
subcommands: [micList, micSet])
|
| 19 |
+
let serviceRoot = CommandDescriptor(
|
| 20 |
+
name: "service",
|
| 21 |
+
abstract: "launchd helper",
|
| 22 |
+
discussion: nil,
|
| 23 |
+
signature: CommandSignature(),
|
| 24 |
+
subcommands: [
|
| 25 |
+
descriptor(for: ServiceInstall.self),
|
| 26 |
+
descriptor(for: ServiceUninstall.self),
|
| 27 |
+
descriptor(for: ServiceStatus.self)
|
| 28 |
+
])
|
| 29 |
+
let doctorDesc = descriptor(for: DoctorCommand.self)
|
| 30 |
+
let setupDesc = descriptor(for: SetupCommand.self)
|
| 31 |
+
let healthDesc = descriptor(for: HealthCommand.self)
|
| 32 |
+
let tailLogDesc = descriptor(for: TailLogCommand.self)
|
| 33 |
+
let startDesc = descriptor(for: StartCommand.self)
|
| 34 |
+
let stopDesc = descriptor(for: StopCommand.self)
|
| 35 |
+
let restartDesc = descriptor(for: RestartCommand.self)
|
| 36 |
+
let statusDesc = descriptor(for: StatusCommand.self)
|
| 37 |
+
|
| 38 |
+
let rootSignature = CommandSignature().withStandardRuntimeFlags()
|
| 39 |
+
let root = CommandDescriptor(
|
| 40 |
+
name: "swabble",
|
| 41 |
+
abstract: "Speech hook daemon",
|
| 42 |
+
discussion: "Local wake-word → SpeechTranscriber → hook",
|
| 43 |
+
signature: rootSignature,
|
| 44 |
+
subcommands: [
|
| 45 |
+
serveDesc,
|
| 46 |
+
transcribeDesc,
|
| 47 |
+
testHookDesc,
|
| 48 |
+
micRoot,
|
| 49 |
+
serviceRoot,
|
| 50 |
+
doctorDesc,
|
| 51 |
+
setupDesc,
|
| 52 |
+
healthDesc,
|
| 53 |
+
tailLogDesc,
|
| 54 |
+
startDesc,
|
| 55 |
+
stopDesc,
|
| 56 |
+
restartDesc,
|
| 57 |
+
statusDesc
|
| 58 |
+
])
|
| 59 |
+
return [root]
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
private static func descriptor(for type: any ParsableCommand.Type) -> CommandDescriptor {
|
| 63 |
+
let sig = CommandSignature.describe(type.init()).withStandardRuntimeFlags()
|
| 64 |
+
return CommandDescriptor(
|
| 65 |
+
name: type.commandDescription.commandName ?? "",
|
| 66 |
+
abstract: type.commandDescription.abstract,
|
| 67 |
+
discussion: type.commandDescription.discussion,
|
| 68 |
+
signature: sig,
|
| 69 |
+
subcommands: [])
|
| 70 |
+
}
|
| 71 |
+
}
|
Swabble/Sources/swabble/Commands/DoctorCommand.swift
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
import Speech
|
| 4 |
+
import Swabble
|
| 5 |
+
|
| 6 |
+
@MainActor
|
| 7 |
+
struct DoctorCommand: ParsableCommand {
|
| 8 |
+
static var commandDescription: CommandDescription {
|
| 9 |
+
CommandDescription(commandName: "doctor", abstract: "Check Speech permission and config")
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
|
| 13 |
+
|
| 14 |
+
init() {}
|
| 15 |
+
init(parsed: ParsedValues) {
|
| 16 |
+
self.init()
|
| 17 |
+
if let cfg = parsed.options["config"]?.last { configPath = cfg }
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
mutating func run() async throws {
|
| 21 |
+
let auth = await SFSpeechRecognizer.authorizationStatus()
|
| 22 |
+
print("Speech auth: \(auth)")
|
| 23 |
+
do {
|
| 24 |
+
_ = try ConfigLoader.load(at: configURL)
|
| 25 |
+
print("Config: OK")
|
| 26 |
+
} catch {
|
| 27 |
+
print("Config missing or invalid; run setup")
|
| 28 |
+
}
|
| 29 |
+
let session = AVCaptureDevice.DiscoverySession(
|
| 30 |
+
deviceTypes: [.microphone, .external],
|
| 31 |
+
mediaType: .audio,
|
| 32 |
+
position: .unspecified)
|
| 33 |
+
print("Mics found: \(session.devices.count)")
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
|
| 37 |
+
}
|
Swabble/Sources/swabble/Commands/HealthCommand.swift
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
|
| 4 |
+
@MainActor
|
| 5 |
+
struct HealthCommand: ParsableCommand {
|
| 6 |
+
static var commandDescription: CommandDescription {
|
| 7 |
+
CommandDescription(commandName: "health", abstract: "Health probe")
|
| 8 |
+
}
|
| 9 |
+
|
| 10 |
+
init() {}
|
| 11 |
+
init(parsed: ParsedValues) {}
|
| 12 |
+
|
| 13 |
+
mutating func run() async throws {
|
| 14 |
+
print("ok")
|
| 15 |
+
}
|
| 16 |
+
}
|
Swabble/Sources/swabble/Commands/MicCommands.swift
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import AVFoundation
|
| 2 |
+
import Commander
|
| 3 |
+
import Foundation
|
| 4 |
+
import Swabble
|
| 5 |
+
|
| 6 |
+
@MainActor
|
| 7 |
+
struct MicCommand: ParsableCommand {
|
| 8 |
+
static var commandDescription: CommandDescription {
|
| 9 |
+
CommandDescription(
|
| 10 |
+
commandName: "mic",
|
| 11 |
+
abstract: "Microphone management",
|
| 12 |
+
subcommands: [MicList.self, MicSet.self])
|
| 13 |
+
}
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
@MainActor
|
| 17 |
+
struct MicList: ParsableCommand {
|
| 18 |
+
static var commandDescription: CommandDescription {
|
| 19 |
+
CommandDescription(commandName: "list", abstract: "List input devices")
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
init() {}
|
| 23 |
+
init(parsed: ParsedValues) {}
|
| 24 |
+
|
| 25 |
+
mutating func run() async throws {
|
| 26 |
+
let session = AVCaptureDevice.DiscoverySession(
|
| 27 |
+
deviceTypes: [.microphone, .external],
|
| 28 |
+
mediaType: .audio,
|
| 29 |
+
position: .unspecified)
|
| 30 |
+
let devices = session.devices
|
| 31 |
+
if devices.isEmpty { print("no audio inputs found"); return }
|
| 32 |
+
for (idx, device) in devices.enumerated() {
|
| 33 |
+
print("[\(idx)] \(device.localizedName)")
|
| 34 |
+
}
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
@MainActor
|
| 39 |
+
struct MicSet: ParsableCommand {
|
| 40 |
+
@Argument(help: "Device index from list") var index: Int = 0
|
| 41 |
+
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
|
| 42 |
+
|
| 43 |
+
static var commandDescription: CommandDescription {
|
| 44 |
+
CommandDescription(commandName: "set", abstract: "Set default input device index")
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
init() {}
|
| 48 |
+
init(parsed: ParsedValues) {
|
| 49 |
+
self.init()
|
| 50 |
+
if let value = parsed.positional.first, let intVal = Int(value) { index = intVal }
|
| 51 |
+
if let cfg = parsed.options["config"]?.last { configPath = cfg }
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
mutating func run() async throws {
|
| 55 |
+
var cfg = try ConfigLoader.load(at: configURL)
|
| 56 |
+
cfg.audio.deviceIndex = index
|
| 57 |
+
try ConfigLoader.save(cfg, at: configURL)
|
| 58 |
+
print("saved device index \(index)")
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
|
| 62 |
+
}
|
Swabble/Sources/swabble/Commands/ServeCommand.swift
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
import Swabble
|
| 4 |
+
import SwabbleKit
|
| 5 |
+
|
| 6 |
+
@available(macOS 26.0, *)
|
| 7 |
+
@MainActor
|
| 8 |
+
struct ServeCommand: ParsableCommand {
|
| 9 |
+
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
|
| 10 |
+
@Flag(name: .long("no-wake"), help: "Disable wake word") var noWake: Bool = false
|
| 11 |
+
|
| 12 |
+
static var commandDescription: CommandDescription {
|
| 13 |
+
CommandDescription(
|
| 14 |
+
commandName: "serve",
|
| 15 |
+
abstract: "Run swabble in the foreground")
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
init() {}
|
| 19 |
+
|
| 20 |
+
init(parsed: ParsedValues) {
|
| 21 |
+
self.init()
|
| 22 |
+
if parsed.flags.contains("noWake") { noWake = true }
|
| 23 |
+
if let cfg = parsed.options["config"]?.last { configPath = cfg }
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
mutating func run() async throws {
|
| 27 |
+
var cfg: SwabbleConfig
|
| 28 |
+
do {
|
| 29 |
+
cfg = try ConfigLoader.load(at: configURL)
|
| 30 |
+
} catch {
|
| 31 |
+
cfg = SwabbleConfig()
|
| 32 |
+
try ConfigLoader.save(cfg, at: configURL)
|
| 33 |
+
}
|
| 34 |
+
if noWake {
|
| 35 |
+
cfg.wake.enabled = false
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
let logger = Logger(level: LogLevel(configValue: cfg.logging.level) ?? .info)
|
| 39 |
+
logger.info("swabble serve starting (wake: \(cfg.wake.enabled ? cfg.wake.word : "disabled"))")
|
| 40 |
+
let pipeline = SpeechPipeline()
|
| 41 |
+
do {
|
| 42 |
+
let stream = try await pipeline.start(
|
| 43 |
+
localeIdentifier: cfg.speech.localeIdentifier,
|
| 44 |
+
etiquette: cfg.speech.etiquetteReplacements)
|
| 45 |
+
for await seg in stream {
|
| 46 |
+
if cfg.wake.enabled {
|
| 47 |
+
guard Self.matchesWake(text: seg.text, cfg: cfg) else { continue }
|
| 48 |
+
}
|
| 49 |
+
let stripped = Self.stripWake(text: seg.text, cfg: cfg)
|
| 50 |
+
let job = HookJob(text: stripped, timestamp: Date())
|
| 51 |
+
let executor = HookExecutor(config: cfg)
|
| 52 |
+
try await executor.run(job: job)
|
| 53 |
+
if cfg.transcripts.enabled {
|
| 54 |
+
await TranscriptsStore.shared.append(text: stripped)
|
| 55 |
+
}
|
| 56 |
+
if seg.isFinal {
|
| 57 |
+
logger.info("final: \(stripped)")
|
| 58 |
+
} else {
|
| 59 |
+
logger.debug("partial: \(stripped)")
|
| 60 |
+
}
|
| 61 |
+
}
|
| 62 |
+
} catch {
|
| 63 |
+
logger.error("serve error: \(error)")
|
| 64 |
+
throw error
|
| 65 |
+
}
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
private var configURL: URL? {
|
| 69 |
+
configPath.map { URL(fileURLWithPath: $0) }
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
private static func matchesWake(text: String, cfg: SwabbleConfig) -> Bool {
|
| 73 |
+
let triggers = [cfg.wake.word] + cfg.wake.aliases
|
| 74 |
+
return WakeWordGate.matchesTextOnly(text: text, triggers: triggers)
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
private static func stripWake(text: String, cfg: SwabbleConfig) -> String {
|
| 78 |
+
let triggers = [cfg.wake.word] + cfg.wake.aliases
|
| 79 |
+
return WakeWordGate.stripWake(text: text, triggers: triggers)
|
| 80 |
+
}
|
| 81 |
+
}
|
Swabble/Sources/swabble/Commands/ServiceCommands.swift
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
|
| 4 |
+
@MainActor
|
| 5 |
+
struct ServiceRootCommand: ParsableCommand {
|
| 6 |
+
static var commandDescription: CommandDescription {
|
| 7 |
+
CommandDescription(
|
| 8 |
+
commandName: "service",
|
| 9 |
+
abstract: "Manage launchd agent",
|
| 10 |
+
subcommands: [ServiceInstall.self, ServiceUninstall.self, ServiceStatus.self])
|
| 11 |
+
}
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
private enum LaunchdHelper {
|
| 15 |
+
static let label = "com.swabble.agent"
|
| 16 |
+
|
| 17 |
+
static var plistURL: URL {
|
| 18 |
+
FileManager.default
|
| 19 |
+
.homeDirectoryForCurrentUser
|
| 20 |
+
.appendingPathComponent("Library/LaunchAgents/\(label).plist")
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
static func writePlist(executable: String) throws {
|
| 24 |
+
let plist: [String: Any] = [
|
| 25 |
+
"Label": label,
|
| 26 |
+
"ProgramArguments": [executable, "serve"],
|
| 27 |
+
"RunAtLoad": true,
|
| 28 |
+
"KeepAlive": true
|
| 29 |
+
]
|
| 30 |
+
let data = try PropertyListSerialization.data(fromPropertyList: plist, format: .xml, options: 0)
|
| 31 |
+
try data.write(to: plistURL)
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
static func removePlist() throws {
|
| 35 |
+
try? FileManager.default.removeItem(at: plistURL)
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
@MainActor
|
| 40 |
+
struct ServiceInstall: ParsableCommand {
|
| 41 |
+
static var commandDescription: CommandDescription {
|
| 42 |
+
CommandDescription(commandName: "install", abstract: "Install user launch agent")
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
mutating func run() async throws {
|
| 46 |
+
let exe = CommandLine.arguments.first ?? "/usr/local/bin/swabble"
|
| 47 |
+
try LaunchdHelper.writePlist(executable: exe)
|
| 48 |
+
print("launchctl load -w \(LaunchdHelper.plistURL.path)")
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
@MainActor
|
| 53 |
+
struct ServiceUninstall: ParsableCommand {
|
| 54 |
+
static var commandDescription: CommandDescription {
|
| 55 |
+
CommandDescription(commandName: "uninstall", abstract: "Remove launch agent")
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
mutating func run() async throws {
|
| 59 |
+
try LaunchdHelper.removePlist()
|
| 60 |
+
print("launchctl bootout gui/$(id -u)/\(LaunchdHelper.label)")
|
| 61 |
+
}
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
@MainActor
|
| 65 |
+
struct ServiceStatus: ParsableCommand {
|
| 66 |
+
static var commandDescription: CommandDescription {
|
| 67 |
+
CommandDescription(commandName: "status", abstract: "Show launch agent status")
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
mutating func run() async throws {
|
| 71 |
+
if FileManager.default.fileExists(atPath: LaunchdHelper.plistURL.path) {
|
| 72 |
+
print("plist present at \(LaunchdHelper.plistURL.path)")
|
| 73 |
+
} else {
|
| 74 |
+
print("launchd plist not installed")
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
}
|
Swabble/Sources/swabble/Commands/SetupCommand.swift
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
import Swabble
|
| 4 |
+
|
| 5 |
+
@MainActor
|
| 6 |
+
struct SetupCommand: ParsableCommand {
|
| 7 |
+
static var commandDescription: CommandDescription {
|
| 8 |
+
CommandDescription(commandName: "setup", abstract: "Write default config")
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
|
| 12 |
+
|
| 13 |
+
init() {}
|
| 14 |
+
init(parsed: ParsedValues) {
|
| 15 |
+
self.init()
|
| 16 |
+
if let cfg = parsed.options["config"]?.last { configPath = cfg }
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
mutating func run() async throws {
|
| 20 |
+
let cfg = SwabbleConfig()
|
| 21 |
+
try ConfigLoader.save(cfg, at: configURL)
|
| 22 |
+
print("wrote config to \(configURL?.path ?? SwabbleConfig.defaultPath.path)")
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
|
| 26 |
+
}
|
Swabble/Sources/swabble/Commands/StartStopCommands.swift
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
|
| 4 |
+
@MainActor
|
| 5 |
+
struct StartCommand: ParsableCommand {
|
| 6 |
+
static var commandDescription: CommandDescription {
|
| 7 |
+
CommandDescription(commandName: "start", abstract: "Start swabble (foreground placeholder)")
|
| 8 |
+
}
|
| 9 |
+
|
| 10 |
+
mutating func run() async throws {
|
| 11 |
+
print("start: launchd helper not implemented; run 'swabble serve' instead")
|
| 12 |
+
}
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
@MainActor
|
| 16 |
+
struct StopCommand: ParsableCommand {
|
| 17 |
+
static var commandDescription: CommandDescription {
|
| 18 |
+
CommandDescription(commandName: "stop", abstract: "Stop swabble (placeholder)")
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
mutating func run() async throws {
|
| 22 |
+
print("stop: launchd helper not implemented yet")
|
| 23 |
+
}
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
@MainActor
|
| 27 |
+
struct RestartCommand: ParsableCommand {
|
| 28 |
+
static var commandDescription: CommandDescription {
|
| 29 |
+
CommandDescription(commandName: "restart", abstract: "Restart swabble (placeholder)")
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
mutating func run() async throws {
|
| 33 |
+
print("restart: launchd helper not implemented yet")
|
| 34 |
+
}
|
| 35 |
+
}
|
Swabble/Sources/swabble/Commands/StatusCommand.swift
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
import Swabble
|
| 4 |
+
|
| 5 |
+
@MainActor
|
| 6 |
+
struct StatusCommand: ParsableCommand {
|
| 7 |
+
static var commandDescription: CommandDescription {
|
| 8 |
+
CommandDescription(commandName: "status", abstract: "Show daemon state")
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
|
| 12 |
+
|
| 13 |
+
init() {}
|
| 14 |
+
init(parsed: ParsedValues) {
|
| 15 |
+
self.init()
|
| 16 |
+
if let cfg = parsed.options["config"]?.last { configPath = cfg }
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
mutating func run() async throws {
|
| 20 |
+
let cfg = try? ConfigLoader.load(at: configURL)
|
| 21 |
+
let wake = cfg?.wake.word ?? "clawd"
|
| 22 |
+
let wakeEnabled = cfg?.wake.enabled ?? false
|
| 23 |
+
let latest = await TranscriptsStore.shared.latest().suffix(3)
|
| 24 |
+
print("wake: \(wakeEnabled ? wake : "disabled")")
|
| 25 |
+
if latest.isEmpty {
|
| 26 |
+
print("transcripts: (none yet)")
|
| 27 |
+
} else {
|
| 28 |
+
print("last transcripts:")
|
| 29 |
+
latest.forEach { print("- \($0)") }
|
| 30 |
+
}
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
|
| 34 |
+
}
|
Swabble/Sources/swabble/Commands/TailLogCommand.swift
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
import Swabble
|
| 4 |
+
|
| 5 |
+
@MainActor
|
| 6 |
+
struct TailLogCommand: ParsableCommand {
|
| 7 |
+
static var commandDescription: CommandDescription {
|
| 8 |
+
CommandDescription(commandName: "tail-log", abstract: "Tail recent transcripts")
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
init() {}
|
| 12 |
+
init(parsed: ParsedValues) {}
|
| 13 |
+
|
| 14 |
+
mutating func run() async throws {
|
| 15 |
+
let latest = await TranscriptsStore.shared.latest()
|
| 16 |
+
for line in latest.suffix(10) {
|
| 17 |
+
print(line)
|
| 18 |
+
}
|
| 19 |
+
}
|
| 20 |
+
}
|
Swabble/Sources/swabble/Commands/TestHookCommand.swift
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
import Swabble
|
| 4 |
+
|
| 5 |
+
@MainActor
|
| 6 |
+
struct TestHookCommand: ParsableCommand {
|
| 7 |
+
@Argument(help: "Text to send to hook") var text: String
|
| 8 |
+
@Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
|
| 9 |
+
|
| 10 |
+
static var commandDescription: CommandDescription {
|
| 11 |
+
CommandDescription(commandName: "test-hook", abstract: "Invoke the configured hook with text")
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
init() {}
|
| 15 |
+
|
| 16 |
+
init(parsed: ParsedValues) {
|
| 17 |
+
self.init()
|
| 18 |
+
if let positional = parsed.positional.first { text = positional }
|
| 19 |
+
if let cfg = parsed.options["config"]?.last { configPath = cfg }
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
mutating func run() async throws {
|
| 23 |
+
let cfg = try ConfigLoader.load(at: configURL)
|
| 24 |
+
let executor = HookExecutor(config: cfg)
|
| 25 |
+
try await executor.run(job: HookJob(text: text, timestamp: Date()))
|
| 26 |
+
print("hook invoked")
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
|
| 30 |
+
}
|
Swabble/Sources/swabble/Commands/TranscribeCommand.swift
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import AVFoundation
|
| 2 |
+
import Commander
|
| 3 |
+
import Foundation
|
| 4 |
+
import Speech
|
| 5 |
+
import Swabble
|
| 6 |
+
|
| 7 |
+
@MainActor
|
| 8 |
+
struct TranscribeCommand: ParsableCommand {
|
| 9 |
+
@Argument(help: "Path to audio/video file") var inputFile: String = ""
|
| 10 |
+
@Option(name: .long("locale"), help: "Locale identifier", parsing: .singleValue) var locale: String = Locale.current
|
| 11 |
+
.identifier
|
| 12 |
+
@Flag(help: "Censor etiquette-sensitive content") var censor: Bool = false
|
| 13 |
+
@Option(name: .long("output"), help: "Output file path") var outputFile: String?
|
| 14 |
+
@Option(name: .long("format"), help: "Output format txt|srt") var format: String = "txt"
|
| 15 |
+
@Option(name: .long("max-length"), help: "Max sentence length for srt") var maxLength: Int = 40
|
| 16 |
+
|
| 17 |
+
static var commandDescription: CommandDescription {
|
| 18 |
+
CommandDescription(
|
| 19 |
+
commandName: "transcribe",
|
| 20 |
+
abstract: "Transcribe a media file locally")
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
init() {}
|
| 24 |
+
|
| 25 |
+
init(parsed: ParsedValues) {
|
| 26 |
+
self.init()
|
| 27 |
+
if let positional = parsed.positional.first { inputFile = positional }
|
| 28 |
+
if let loc = parsed.options["locale"]?.last { locale = loc }
|
| 29 |
+
if parsed.flags.contains("censor") { censor = true }
|
| 30 |
+
if let out = parsed.options["output"]?.last { outputFile = out }
|
| 31 |
+
if let fmt = parsed.options["format"]?.last { format = fmt }
|
| 32 |
+
if let len = parsed.options["maxLength"]?.last, let intVal = Int(len) { maxLength = intVal }
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
mutating func run() async throws {
|
| 36 |
+
let fileURL = URL(fileURLWithPath: inputFile)
|
| 37 |
+
let audioFile = try AVAudioFile(forReading: fileURL)
|
| 38 |
+
|
| 39 |
+
let outputFormat = OutputFormat(rawValue: format) ?? .txt
|
| 40 |
+
|
| 41 |
+
let transcriber = SpeechTranscriber(
|
| 42 |
+
locale: Locale(identifier: locale),
|
| 43 |
+
transcriptionOptions: censor ? [.etiquetteReplacements] : [],
|
| 44 |
+
reportingOptions: [],
|
| 45 |
+
attributeOptions: outputFormat.needsAudioTimeRange ? [.audioTimeRange] : [])
|
| 46 |
+
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
| 47 |
+
try await analyzer.start(inputAudioFile: audioFile, finishAfterFile: true)
|
| 48 |
+
|
| 49 |
+
var transcript: AttributedString = ""
|
| 50 |
+
for try await result in transcriber.results {
|
| 51 |
+
transcript += result.text
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
let output = outputFormat.text(for: transcript, maxLength: maxLength)
|
| 55 |
+
if let path = outputFile {
|
| 56 |
+
try output.write(to: URL(fileURLWithPath: path), atomically: false, encoding: .utf8)
|
| 57 |
+
} else {
|
| 58 |
+
print(output)
|
| 59 |
+
}
|
| 60 |
+
}
|
| 61 |
+
}
|
Swabble/Sources/swabble/main.swift
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Commander
|
| 2 |
+
import Foundation
|
| 3 |
+
|
| 4 |
+
@available(macOS 26.0, *)
|
| 5 |
+
@MainActor
|
| 6 |
+
private func runCLI() async -> Int32 {
|
| 7 |
+
do {
|
| 8 |
+
let descriptors = CLIRegistry.descriptors
|
| 9 |
+
let program = Program(descriptors: descriptors)
|
| 10 |
+
let invocation = try program.resolve(argv: CommandLine.arguments)
|
| 11 |
+
try await dispatch(invocation: invocation)
|
| 12 |
+
return 0
|
| 13 |
+
} catch {
|
| 14 |
+
fputs("error: \(error)\n", stderr)
|
| 15 |
+
return 1
|
| 16 |
+
}
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
@available(macOS 26.0, *)
|
| 20 |
+
@MainActor
|
| 21 |
+
private func dispatch(invocation: CommandInvocation) async throws {
|
| 22 |
+
let parsed = invocation.parsedValues
|
| 23 |
+
let path = invocation.path
|
| 24 |
+
guard let first = path.first else { throw CommanderProgramError.missingCommand }
|
| 25 |
+
|
| 26 |
+
switch first {
|
| 27 |
+
case "swabble":
|
| 28 |
+
try await dispatchSwabble(parsed: parsed, path: path)
|
| 29 |
+
default:
|
| 30 |
+
throw CommanderProgramError.unknownCommand(first)
|
| 31 |
+
}
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
@available(macOS 26.0, *)
|
| 35 |
+
@MainActor
|
| 36 |
+
private func dispatchSwabble(parsed: ParsedValues, path: [String]) async throws {
|
| 37 |
+
let sub = try subcommand(path, index: 1, command: "swabble")
|
| 38 |
+
switch sub {
|
| 39 |
+
case "mic":
|
| 40 |
+
try await dispatchMic(parsed: parsed, path: path)
|
| 41 |
+
case "service":
|
| 42 |
+
try await dispatchService(path: path)
|
| 43 |
+
default:
|
| 44 |
+
let handlers = swabbleHandlers(parsed: parsed)
|
| 45 |
+
guard let handler = handlers[sub] else {
|
| 46 |
+
throw CommanderProgramError.unknownSubcommand(command: "swabble", name: sub)
|
| 47 |
+
}
|
| 48 |
+
try await handler()
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
@available(macOS 26.0, *)
|
| 53 |
+
@MainActor
|
| 54 |
+
private func swabbleHandlers(parsed: ParsedValues) -> [String: () async throws -> Void] {
|
| 55 |
+
[
|
| 56 |
+
"serve": {
|
| 57 |
+
var cmd = ServeCommand(parsed: parsed)
|
| 58 |
+
try await cmd.run()
|
| 59 |
+
},
|
| 60 |
+
"transcribe": {
|
| 61 |
+
var cmd = TranscribeCommand(parsed: parsed)
|
| 62 |
+
try await cmd.run()
|
| 63 |
+
},
|
| 64 |
+
"test-hook": {
|
| 65 |
+
var cmd = TestHookCommand(parsed: parsed)
|
| 66 |
+
try await cmd.run()
|
| 67 |
+
},
|
| 68 |
+
"doctor": {
|
| 69 |
+
var cmd = DoctorCommand(parsed: parsed)
|
| 70 |
+
try await cmd.run()
|
| 71 |
+
},
|
| 72 |
+
"setup": {
|
| 73 |
+
var cmd = SetupCommand(parsed: parsed)
|
| 74 |
+
try await cmd.run()
|
| 75 |
+
},
|
| 76 |
+
"health": {
|
| 77 |
+
var cmd = HealthCommand(parsed: parsed)
|
| 78 |
+
try await cmd.run()
|
| 79 |
+
},
|
| 80 |
+
"tail-log": {
|
| 81 |
+
var cmd = TailLogCommand(parsed: parsed)
|
| 82 |
+
try await cmd.run()
|
| 83 |
+
},
|
| 84 |
+
"start": {
|
| 85 |
+
var cmd = StartCommand()
|
| 86 |
+
try await cmd.run()
|
| 87 |
+
},
|
| 88 |
+
"stop": {
|
| 89 |
+
var cmd = StopCommand()
|
| 90 |
+
try await cmd.run()
|
| 91 |
+
},
|
| 92 |
+
"restart": {
|
| 93 |
+
var cmd = RestartCommand()
|
| 94 |
+
try await cmd.run()
|
| 95 |
+
},
|
| 96 |
+
"status": {
|
| 97 |
+
var cmd = StatusCommand()
|
| 98 |
+
try await cmd.run()
|
| 99 |
+
}
|
| 100 |
+
]
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
@available(macOS 26.0, *)
|
| 104 |
+
@MainActor
|
| 105 |
+
private func dispatchMic(parsed: ParsedValues, path: [String]) async throws {
|
| 106 |
+
let micSub = try subcommand(path, index: 2, command: "mic")
|
| 107 |
+
switch micSub {
|
| 108 |
+
case "list":
|
| 109 |
+
var cmd = MicList(parsed: parsed)
|
| 110 |
+
try await cmd.run()
|
| 111 |
+
case "set":
|
| 112 |
+
var cmd = MicSet(parsed: parsed)
|
| 113 |
+
try await cmd.run()
|
| 114 |
+
default:
|
| 115 |
+
throw CommanderProgramError.unknownSubcommand(command: "mic", name: micSub)
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
@available(macOS 26.0, *)
|
| 120 |
+
@MainActor
|
| 121 |
+
private func dispatchService(path: [String]) async throws {
|
| 122 |
+
let svcSub = try subcommand(path, index: 2, command: "service")
|
| 123 |
+
switch svcSub {
|
| 124 |
+
case "install":
|
| 125 |
+
var cmd = ServiceInstall()
|
| 126 |
+
try await cmd.run()
|
| 127 |
+
case "uninstall":
|
| 128 |
+
var cmd = ServiceUninstall()
|
| 129 |
+
try await cmd.run()
|
| 130 |
+
case "status":
|
| 131 |
+
var cmd = ServiceStatus()
|
| 132 |
+
try await cmd.run()
|
| 133 |
+
default:
|
| 134 |
+
throw CommanderProgramError.unknownSubcommand(command: "service", name: svcSub)
|
| 135 |
+
}
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
private func subcommand(_ path: [String], index: Int, command: String) throws -> String {
|
| 139 |
+
guard path.count > index else {
|
| 140 |
+
throw CommanderProgramError.missingSubcommand(command: command)
|
| 141 |
+
}
|
| 142 |
+
return path[index]
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
if #available(macOS 26.0, *) {
|
| 146 |
+
let exitCode = await runCLI()
|
| 147 |
+
exit(exitCode)
|
| 148 |
+
} else {
|
| 149 |
+
fputs("error: swabble requires macOS 26 or newer\n", stderr)
|
| 150 |
+
exit(1)
|
| 151 |
+
}
|
Swabble/Tests/SwabbleKitTests/WakeWordGateTests.swift
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Foundation
|
| 2 |
+
import SwabbleKit
|
| 3 |
+
import Testing
|
| 4 |
+
|
| 5 |
+
@Suite struct WakeWordGateTests {
|
| 6 |
+
@Test func matchRequiresGapAfterTrigger() {
|
| 7 |
+
let transcript = "hey clawd do thing"
|
| 8 |
+
let segments = makeSegments(
|
| 9 |
+
transcript: transcript,
|
| 10 |
+
words: [
|
| 11 |
+
("hey", 0.0, 0.1),
|
| 12 |
+
("clawd", 0.2, 0.1),
|
| 13 |
+
("do", 0.35, 0.1),
|
| 14 |
+
("thing", 0.5, 0.1),
|
| 15 |
+
])
|
| 16 |
+
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
|
| 17 |
+
#expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config) == nil)
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
@Test func matchAllowsGapAndExtractsCommand() {
|
| 21 |
+
let transcript = "hey clawd do thing"
|
| 22 |
+
let segments = makeSegments(
|
| 23 |
+
transcript: transcript,
|
| 24 |
+
words: [
|
| 25 |
+
("hey", 0.0, 0.1),
|
| 26 |
+
("clawd", 0.2, 0.1),
|
| 27 |
+
("do", 0.9, 0.1),
|
| 28 |
+
("thing", 1.1, 0.1),
|
| 29 |
+
])
|
| 30 |
+
let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
|
| 31 |
+
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
|
| 32 |
+
#expect(match?.command == "do thing")
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
@Test func matchHandlesMultiWordTriggers() {
|
| 36 |
+
let transcript = "hey clawd do it"
|
| 37 |
+
let segments = makeSegments(
|
| 38 |
+
transcript: transcript,
|
| 39 |
+
words: [
|
| 40 |
+
("hey", 0.0, 0.1),
|
| 41 |
+
("clawd", 0.2, 0.1),
|
| 42 |
+
("do", 0.8, 0.1),
|
| 43 |
+
("it", 1.0, 0.1),
|
| 44 |
+
])
|
| 45 |
+
let config = WakeWordGateConfig(triggers: ["hey clawd"], minPostTriggerGap: 0.3)
|
| 46 |
+
let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
|
| 47 |
+
#expect(match?.command == "do it")
|
| 48 |
+
}
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
private func makeSegments(
|
| 52 |
+
transcript: String,
|
| 53 |
+
words: [(String, TimeInterval, TimeInterval)])
|
| 54 |
+
-> [WakeWordSegment] {
|
| 55 |
+
var searchStart = transcript.startIndex
|
| 56 |
+
var output: [WakeWordSegment] = []
|
| 57 |
+
for (word, start, duration) in words {
|
| 58 |
+
let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
|
| 59 |
+
output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
|
| 60 |
+
if let range { searchStart = range.upperBound }
|
| 61 |
+
}
|
| 62 |
+
return output
|
| 63 |
+
}
|
Swabble/Tests/swabbleTests/ConfigTests.swift
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import Foundation
|
| 2 |
+
import Testing
|
| 3 |
+
@testable import Swabble
|
| 4 |
+
|
| 5 |
+
@Test
|
| 6 |
+
func configRoundTrip() throws {
|
| 7 |
+
var cfg = SwabbleConfig()
|
| 8 |
+
cfg.wake.word = "robot"
|
| 9 |
+
let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".json")
|
| 10 |
+
defer { try? FileManager.default.removeItem(at: url) }
|
| 11 |
+
|
| 12 |
+
try ConfigLoader.save(cfg, at: url)
|
| 13 |
+
let loaded = try ConfigLoader.load(at: url)
|
| 14 |
+
#expect(loaded.wake.word == "robot")
|
| 15 |
+
#expect(loaded.hook.prefix.contains("Voice swabble"))
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
@Test
|
| 19 |
+
func configMissingThrows() {
|
| 20 |
+
#expect(throws: ConfigError.missingConfig) {
|
| 21 |
+
_ = try ConfigLoader.load(at: FileManager.default.temporaryDirectory.appendingPathComponent("nope.json"))
|
| 22 |
+
}
|
| 23 |
+
}
|
Swabble/docs/spec.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# swabble — macOS 26 speech hook daemon (Swift 6.2)
|
| 2 |
+
|
| 3 |
+
Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript. Shared wake-gate utilities live in `SwabbleKit` for reuse by other apps (iOS/macOS).
|
| 4 |
+
|
| 5 |
+
## Requirements
|
| 6 |
+
- macOS 26+, Swift 6.2, Speech.framework with on-device assets.
|
| 7 |
+
- Local only; no network calls during transcription.
|
| 8 |
+
- Wake word gating (default "clawd" plus aliases) with bypass flag `--no-wake`.
|
| 9 |
+
- `SwabbleKit` target (multi-platform) providing wake-word gating helpers that can use speech segment timing to require a post-trigger gap.
|
| 10 |
+
- Hook execution with cooldown, min_chars, timeout, prefix, env vars.
|
| 11 |
+
- Simple config at `~/.config/swabble/config.json` (JSON, Codable) — no TOML.
|
| 12 |
+
- CLI implemented with Commander (SwiftPM package `steipete/Commander`); core types are available via the SwiftPM library product `Swabble` for embedding.
|
| 13 |
+
- Foreground `serve`; later launchd helper for start/stop/restart.
|
| 14 |
+
- File transcription command emitting txt or srt.
|
| 15 |
+
- Basic status/health surfaces and mic selection stubs.
|
| 16 |
+
|
| 17 |
+
## Architecture
|
| 18 |
+
- **CLI layer (Commander)**: Root command `swabble` with subcommands `serve`, `transcribe`, `test-hook`, `mic list|set`, `doctor`, `health`, `tail-log`. Runtime flags from Commander (`-v/--verbose`, `--json-output`, `--log-level`). Custom `--config` path applies everywhere.
|
| 19 |
+
- **Config**: `SwabbleConfig` Codable. Fields: audio device name/index, wake (enabled/word/aliases/sensitivity placeholder), hook (command/args/prefix/cooldown/min_chars/timeout/env), logging (level, format), transcripts (enabled, max kept), speech (locale, enableEtiquetteReplacements flag). Stored JSON; default written by `setup`.
|
| 20 |
+
- **Audio + Speech pipeline**: `SpeechPipeline` wraps `AVAudioEngine` input → `SpeechAnalyzer` with `SpeechTranscriber` module. Emits partial/final transcripts via async stream. Requests `.audioTimeRange` when transcripts enabled. Handles Speech permission and asset download prompts ahead of capture.
|
| 21 |
+
- **Wake gate**: CLI currently uses text-only keyword match; shared `SwabbleKit` gate can enforce a minimum pause between the wake word and the next token when speech segments are available. `--no-wake` disables gating.
|
| 22 |
+
- **Hook executor**: async `HookExecutor` spawns `Process` with configured args, prefix substitution `${hostname}`. Enforces cooldown + timeout; injects env `SWABBLE_TEXT`, `SWABBLE_PREFIX` plus user env map.
|
| 23 |
+
- **Transcripts store**: in-memory ring buffer; optional persisted JSON lines under `~/Library/Application Support/swabble/transcripts.log`.
|
| 24 |
+
- **Logging**: simple structured logger to stderr; respects log level.
|
| 25 |
+
|
| 26 |
+
## Out of scope (initial cut)
|
| 27 |
+
- Model management (Speech handles assets).
|
| 28 |
+
- Launchd helper (planned follow-up).
|
| 29 |
+
- Advanced wake-word detector (segment-aware gate now lives in `SwabbleKit`; CLI still text-only until segment timing is plumbed through).
|
| 30 |
+
|
| 31 |
+
## Open decisions
|
| 32 |
+
- Whether to expose a UNIX control socket for `status`/`health` (currently planned as stdin/out direct calls).
|
| 33 |
+
- Hook redaction (PII) parity with brabble — placeholder boolean, no implementation yet.
|
Swabble/scripts/format.sh
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
| 4 |
+
CONFIG="${ROOT}/.swiftformat"
|
| 5 |
+
swiftformat --config "$CONFIG" "$ROOT/Sources"
|
Swabble/scripts/lint.sh
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
| 4 |
+
CONFIG="${ROOT}/.swiftlint.yml"
|
| 5 |
+
if ! command -v swiftlint >/dev/null; then
|
| 6 |
+
echo "swiftlint not installed" >&2
|
| 7 |
+
exit 1
|
| 8 |
+
fi
|
| 9 |
+
swiftlint --config "$CONFIG"
|
apps/android/.gitignore
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.gradle/
|
| 2 |
+
**/build/
|
| 3 |
+
local.properties
|
| 4 |
+
.idea/
|
| 5 |
+
**/*.iml
|
apps/android/README.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## OpenClaw Node (Android) (internal)
|
| 2 |
+
|
| 3 |
+
Modern Android node app: connects to the **Gateway WebSocket** (`_openclaw-gw._tcp`) and exposes **Canvas + Chat + Camera**.
|
| 4 |
+
|
| 5 |
+
Notes:
|
| 6 |
+
- The node keeps the connection alive via a **foreground service** (persistent notification with a Disconnect action).
|
| 7 |
+
- Chat always uses the shared session key **`main`** (same session across iOS/macOS/WebChat/Android).
|
| 8 |
+
- Supports modern Android only (`minSdk 31`, Kotlin + Jetpack Compose).
|
| 9 |
+
|
| 10 |
+
## Open in Android Studio
|
| 11 |
+
- Open the folder `apps/android`.
|
| 12 |
+
|
| 13 |
+
## Build / Run
|
| 14 |
+
|
| 15 |
+
```bash
|
| 16 |
+
cd apps/android
|
| 17 |
+
./gradlew :app:assembleDebug
|
| 18 |
+
./gradlew :app:installDebug
|
| 19 |
+
./gradlew :app:testDebugUnitTest
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
`gradlew` auto-detects the Android SDK at `~/Library/Android/sdk` (macOS default) if `ANDROID_SDK_ROOT` / `ANDROID_HOME` are unset.
|
| 23 |
+
|
| 24 |
+
## Connect / Pair
|
| 25 |
+
|
| 26 |
+
1) Start the gateway (on your “master” machine):
|
| 27 |
+
```bash
|
| 28 |
+
pnpm openclaw gateway --port 18789 --verbose
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
2) In the Android app:
|
| 32 |
+
- Open **Settings**
|
| 33 |
+
- Either select a discovered gateway under **Discovered Gateways**, or use **Advanced → Manual Gateway** (host + port).
|
| 34 |
+
|
| 35 |
+
3) Approve pairing (on the gateway machine):
|
| 36 |
+
```bash
|
| 37 |
+
openclaw nodes pending
|
| 38 |
+
openclaw nodes approve <requestId>
|
| 39 |
+
```
|
| 40 |
+
|
| 41 |
+
More details: `docs/platforms/android.md`.
|
| 42 |
+
|
| 43 |
+
## Permissions
|
| 44 |
+
|
| 45 |
+
- Discovery:
|
| 46 |
+
- Android 13+ (`API 33+`): `NEARBY_WIFI_DEVICES`
|
| 47 |
+
- Android 12 and below: `ACCESS_FINE_LOCATION` (required for NSD scanning)
|
| 48 |
+
- Foreground service notification (Android 13+): `POST_NOTIFICATIONS`
|
| 49 |
+
- Camera:
|
| 50 |
+
- `CAMERA` for `camera.snap` and `camera.clip`
|
| 51 |
+
- `RECORD_AUDIO` for `camera.clip` when `includeAudio=true`
|
apps/android/app/build.gradle.kts
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import com.android.build.api.variant.impl.VariantOutputImpl
|
| 2 |
+
|
| 3 |
+
plugins {
|
| 4 |
+
id("com.android.application")
|
| 5 |
+
id("org.jetbrains.kotlin.android")
|
| 6 |
+
id("org.jetbrains.kotlin.plugin.compose")
|
| 7 |
+
id("org.jetbrains.kotlin.plugin.serialization")
|
| 8 |
+
}
|
| 9 |
+
|
| 10 |
+
android {
|
| 11 |
+
namespace = "ai.openclaw.android"
|
| 12 |
+
compileSdk = 36
|
| 13 |
+
|
| 14 |
+
sourceSets {
|
| 15 |
+
getByName("main") {
|
| 16 |
+
assets.srcDir(file("../../shared/OpenClawKit/Sources/OpenClawKit/Resources"))
|
| 17 |
+
}
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
defaultConfig {
|
| 21 |
+
applicationId = "ai.openclaw.android"
|
| 22 |
+
minSdk = 31
|
| 23 |
+
targetSdk = 36
|
| 24 |
+
versionCode = 202601290
|
| 25 |
+
versionName = "2026.1.30"
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
buildTypes {
|
| 29 |
+
release {
|
| 30 |
+
isMinifyEnabled = false
|
| 31 |
+
}
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
buildFeatures {
|
| 35 |
+
compose = true
|
| 36 |
+
buildConfig = true
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
compileOptions {
|
| 40 |
+
sourceCompatibility = JavaVersion.VERSION_17
|
| 41 |
+
targetCompatibility = JavaVersion.VERSION_17
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
packaging {
|
| 45 |
+
resources {
|
| 46 |
+
excludes += "/META-INF/{AL2.0,LGPL2.1}"
|
| 47 |
+
}
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
lint {
|
| 51 |
+
disable += setOf("IconLauncherShape")
|
| 52 |
+
warningsAsErrors = true
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
testOptions {
|
| 56 |
+
unitTests.isIncludeAndroidResources = true
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
androidComponents {
|
| 61 |
+
onVariants { variant ->
|
| 62 |
+
variant.outputs
|
| 63 |
+
.filterIsInstance<VariantOutputImpl>()
|
| 64 |
+
.forEach { output ->
|
| 65 |
+
val versionName = output.versionName.orNull ?: "0"
|
| 66 |
+
val buildType = variant.buildType
|
| 67 |
+
|
| 68 |
+
val outputFileName = "openclaw-${versionName}-${buildType}.apk"
|
| 69 |
+
output.outputFileName = outputFileName
|
| 70 |
+
}
|
| 71 |
+
}
|
| 72 |
+
}
|
| 73 |
+
kotlin {
|
| 74 |
+
compilerOptions {
|
| 75 |
+
jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17)
|
| 76 |
+
allWarningsAsErrors.set(true)
|
| 77 |
+
}
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
dependencies {
|
| 81 |
+
val composeBom = platform("androidx.compose:compose-bom:2025.12.00")
|
| 82 |
+
implementation(composeBom)
|
| 83 |
+
androidTestImplementation(composeBom)
|
| 84 |
+
|
| 85 |
+
implementation("androidx.core:core-ktx:1.17.0")
|
| 86 |
+
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0")
|
| 87 |
+
implementation("androidx.activity:activity-compose:1.12.2")
|
| 88 |
+
implementation("androidx.webkit:webkit:1.15.0")
|
| 89 |
+
|
| 90 |
+
implementation("androidx.compose.ui:ui")
|
| 91 |
+
implementation("androidx.compose.ui:ui-tooling-preview")
|
| 92 |
+
implementation("androidx.compose.material3:material3")
|
| 93 |
+
implementation("androidx.compose.material:material-icons-extended")
|
| 94 |
+
implementation("androidx.navigation:navigation-compose:2.9.6")
|
| 95 |
+
|
| 96 |
+
debugImplementation("androidx.compose.ui:ui-tooling")
|
| 97 |
+
|
| 98 |
+
// Material Components (XML theme + resources)
|
| 99 |
+
implementation("com.google.android.material:material:1.13.0")
|
| 100 |
+
|
| 101 |
+
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.10.2")
|
| 102 |
+
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0")
|
| 103 |
+
|
| 104 |
+
implementation("androidx.security:security-crypto:1.1.0")
|
| 105 |
+
implementation("androidx.exifinterface:exifinterface:1.4.2")
|
| 106 |
+
implementation("com.squareup.okhttp3:okhttp:5.3.2")
|
| 107 |
+
|
| 108 |
+
// CameraX (for node.invoke camera.* parity)
|
| 109 |
+
implementation("androidx.camera:camera-core:1.5.2")
|
| 110 |
+
implementation("androidx.camera:camera-camera2:1.5.2")
|
| 111 |
+
implementation("androidx.camera:camera-lifecycle:1.5.2")
|
| 112 |
+
implementation("androidx.camera:camera-video:1.5.2")
|
| 113 |
+
implementation("androidx.camera:camera-view:1.5.2")
|
| 114 |
+
|
| 115 |
+
// Unicast DNS-SD (Wide-Area Bonjour) for tailnet discovery domains.
|
| 116 |
+
implementation("dnsjava:dnsjava:3.6.4")
|
| 117 |
+
|
| 118 |
+
testImplementation("junit:junit:4.13.2")
|
| 119 |
+
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
|
| 120 |
+
testImplementation("io.kotest:kotest-runner-junit5-jvm:6.0.7")
|
| 121 |
+
testImplementation("io.kotest:kotest-assertions-core-jvm:6.0.7")
|
| 122 |
+
testImplementation("org.robolectric:robolectric:4.16")
|
| 123 |
+
testRuntimeOnly("org.junit.vintage:junit-vintage-engine:6.0.2")
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
tasks.withType<Test>().configureEach {
|
| 127 |
+
useJUnitPlatform()
|
| 128 |
+
}
|
apps/android/app/src/main/AndroidManifest.xml
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
| 2 |
+
<uses-permission android:name="android.permission.INTERNET" />
|
| 3 |
+
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
| 4 |
+
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
| 5 |
+
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
|
| 6 |
+
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
| 7 |
+
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
|
| 8 |
+
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
| 9 |
+
<uses-permission
|
| 10 |
+
android:name="android.permission.NEARBY_WIFI_DEVICES"
|
| 11 |
+
android:usesPermissionFlags="neverForLocation" />
|
| 12 |
+
<uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
|
| 13 |
+
<uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
|
| 14 |
+
<uses-permission android:name="android.permission.ACCESS_BACKGROUND_LOCATION" />
|
| 15 |
+
<uses-permission android:name="android.permission.CAMERA" />
|
| 16 |
+
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
| 17 |
+
<uses-permission android:name="android.permission.SEND_SMS" />
|
| 18 |
+
<uses-feature
|
| 19 |
+
android:name="android.hardware.camera"
|
| 20 |
+
android:required="false" />
|
| 21 |
+
<uses-feature
|
| 22 |
+
android:name="android.hardware.telephony"
|
| 23 |
+
android:required="false" />
|
| 24 |
+
|
| 25 |
+
<application
|
| 26 |
+
android:name=".NodeApp"
|
| 27 |
+
android:allowBackup="true"
|
| 28 |
+
android:dataExtractionRules="@xml/data_extraction_rules"
|
| 29 |
+
android:fullBackupContent="@xml/backup_rules"
|
| 30 |
+
android:icon="@mipmap/ic_launcher"
|
| 31 |
+
android:roundIcon="@mipmap/ic_launcher_round"
|
| 32 |
+
android:label="@string/app_name"
|
| 33 |
+
android:supportsRtl="true"
|
| 34 |
+
android:networkSecurityConfig="@xml/network_security_config"
|
| 35 |
+
android:theme="@style/Theme.OpenClawNode">
|
| 36 |
+
<service
|
| 37 |
+
android:name=".NodeForegroundService"
|
| 38 |
+
android:exported="false"
|
| 39 |
+
android:foregroundServiceType="dataSync|microphone|mediaProjection" />
|
| 40 |
+
<activity
|
| 41 |
+
android:name=".MainActivity"
|
| 42 |
+
android:exported="true">
|
| 43 |
+
<intent-filter>
|
| 44 |
+
<action android:name="android.intent.action.MAIN" />
|
| 45 |
+
<category android:name="android.intent.category.LAUNCHER" />
|
| 46 |
+
</intent-filter>
|
| 47 |
+
</activity>
|
| 48 |
+
</application>
|
| 49 |
+
</manifest>
|
apps/android/app/src/main/java/ai/openclaw/android/CameraHudState.kt
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
enum class CameraHudKind {
|
| 4 |
+
Photo,
|
| 5 |
+
Recording,
|
| 6 |
+
Success,
|
| 7 |
+
Error,
|
| 8 |
+
}
|
| 9 |
+
|
| 10 |
+
data class CameraHudState(
|
| 11 |
+
val token: Long,
|
| 12 |
+
val kind: CameraHudKind,
|
| 13 |
+
val message: String,
|
| 14 |
+
)
|
apps/android/app/src/main/java/ai/openclaw/android/DeviceNames.kt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
import android.content.Context
|
| 4 |
+
import android.os.Build
|
| 5 |
+
import android.provider.Settings
|
| 6 |
+
|
| 7 |
+
object DeviceNames {
|
| 8 |
+
fun bestDefaultNodeName(context: Context): String {
|
| 9 |
+
val deviceName =
|
| 10 |
+
runCatching {
|
| 11 |
+
Settings.Global.getString(context.contentResolver, "device_name")
|
| 12 |
+
}
|
| 13 |
+
.getOrNull()
|
| 14 |
+
?.trim()
|
| 15 |
+
.orEmpty()
|
| 16 |
+
|
| 17 |
+
if (deviceName.isNotEmpty()) return deviceName
|
| 18 |
+
|
| 19 |
+
val model =
|
| 20 |
+
listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() })
|
| 21 |
+
.joinToString(" ")
|
| 22 |
+
.trim()
|
| 23 |
+
|
| 24 |
+
return model.ifEmpty { "Android Node" }
|
| 25 |
+
}
|
| 26 |
+
}
|
apps/android/app/src/main/java/ai/openclaw/android/LocationMode.kt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
enum class LocationMode(val rawValue: String) {
|
| 4 |
+
Off("off"),
|
| 5 |
+
WhileUsing("whileUsing"),
|
| 6 |
+
Always("always"),
|
| 7 |
+
;
|
| 8 |
+
|
| 9 |
+
companion object {
|
| 10 |
+
fun fromRawValue(raw: String?): LocationMode {
|
| 11 |
+
val normalized = raw?.trim()?.lowercase()
|
| 12 |
+
return entries.firstOrNull { it.rawValue.lowercase() == normalized } ?: Off
|
| 13 |
+
}
|
| 14 |
+
}
|
| 15 |
+
}
|
apps/android/app/src/main/java/ai/openclaw/android/MainActivity.kt
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
import android.Manifest
|
| 4 |
+
import android.content.pm.ApplicationInfo
|
| 5 |
+
import android.os.Bundle
|
| 6 |
+
import android.os.Build
|
| 7 |
+
import android.view.WindowManager
|
| 8 |
+
import android.webkit.WebView
|
| 9 |
+
import androidx.activity.ComponentActivity
|
| 10 |
+
import androidx.activity.compose.setContent
|
| 11 |
+
import androidx.activity.viewModels
|
| 12 |
+
import androidx.compose.material3.Surface
|
| 13 |
+
import androidx.compose.ui.Modifier
|
| 14 |
+
import androidx.core.content.ContextCompat
|
| 15 |
+
import androidx.core.view.WindowCompat
|
| 16 |
+
import androidx.core.view.WindowInsetsCompat
|
| 17 |
+
import androidx.core.view.WindowInsetsControllerCompat
|
| 18 |
+
import androidx.lifecycle.Lifecycle
|
| 19 |
+
import androidx.lifecycle.lifecycleScope
|
| 20 |
+
import androidx.lifecycle.repeatOnLifecycle
|
| 21 |
+
import ai.openclaw.android.ui.RootScreen
|
| 22 |
+
import ai.openclaw.android.ui.OpenClawTheme
|
| 23 |
+
import kotlinx.coroutines.launch
|
| 24 |
+
|
| 25 |
+
class MainActivity : ComponentActivity() {
|
| 26 |
+
private val viewModel: MainViewModel by viewModels()
|
| 27 |
+
private lateinit var permissionRequester: PermissionRequester
|
| 28 |
+
private lateinit var screenCaptureRequester: ScreenCaptureRequester
|
| 29 |
+
|
| 30 |
+
override fun onCreate(savedInstanceState: Bundle?) {
|
| 31 |
+
super.onCreate(savedInstanceState)
|
| 32 |
+
val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0
|
| 33 |
+
WebView.setWebContentsDebuggingEnabled(isDebuggable)
|
| 34 |
+
applyImmersiveMode()
|
| 35 |
+
requestDiscoveryPermissionsIfNeeded()
|
| 36 |
+
requestNotificationPermissionIfNeeded()
|
| 37 |
+
NodeForegroundService.start(this)
|
| 38 |
+
permissionRequester = PermissionRequester(this)
|
| 39 |
+
screenCaptureRequester = ScreenCaptureRequester(this)
|
| 40 |
+
viewModel.camera.attachLifecycleOwner(this)
|
| 41 |
+
viewModel.camera.attachPermissionRequester(permissionRequester)
|
| 42 |
+
viewModel.sms.attachPermissionRequester(permissionRequester)
|
| 43 |
+
viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester)
|
| 44 |
+
viewModel.screenRecorder.attachPermissionRequester(permissionRequester)
|
| 45 |
+
|
| 46 |
+
lifecycleScope.launch {
|
| 47 |
+
repeatOnLifecycle(Lifecycle.State.STARTED) {
|
| 48 |
+
viewModel.preventSleep.collect { enabled ->
|
| 49 |
+
if (enabled) {
|
| 50 |
+
window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
| 51 |
+
} else {
|
| 52 |
+
window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
setContent {
|
| 59 |
+
OpenClawTheme {
|
| 60 |
+
Surface(modifier = Modifier) {
|
| 61 |
+
RootScreen(viewModel = viewModel)
|
| 62 |
+
}
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
override fun onResume() {
|
| 68 |
+
super.onResume()
|
| 69 |
+
applyImmersiveMode()
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
override fun onWindowFocusChanged(hasFocus: Boolean) {
|
| 73 |
+
super.onWindowFocusChanged(hasFocus)
|
| 74 |
+
if (hasFocus) {
|
| 75 |
+
applyImmersiveMode()
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
override fun onStart() {
|
| 80 |
+
super.onStart()
|
| 81 |
+
viewModel.setForeground(true)
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
override fun onStop() {
|
| 85 |
+
viewModel.setForeground(false)
|
| 86 |
+
super.onStop()
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
private fun applyImmersiveMode() {
|
| 90 |
+
WindowCompat.setDecorFitsSystemWindows(window, false)
|
| 91 |
+
val controller = WindowInsetsControllerCompat(window, window.decorView)
|
| 92 |
+
controller.systemBarsBehavior =
|
| 93 |
+
WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE
|
| 94 |
+
controller.hide(WindowInsetsCompat.Type.systemBars())
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
private fun requestDiscoveryPermissionsIfNeeded() {
|
| 98 |
+
if (Build.VERSION.SDK_INT >= 33) {
|
| 99 |
+
val ok =
|
| 100 |
+
ContextCompat.checkSelfPermission(
|
| 101 |
+
this,
|
| 102 |
+
Manifest.permission.NEARBY_WIFI_DEVICES,
|
| 103 |
+
) == android.content.pm.PackageManager.PERMISSION_GRANTED
|
| 104 |
+
if (!ok) {
|
| 105 |
+
requestPermissions(arrayOf(Manifest.permission.NEARBY_WIFI_DEVICES), 100)
|
| 106 |
+
}
|
| 107 |
+
} else {
|
| 108 |
+
val ok =
|
| 109 |
+
ContextCompat.checkSelfPermission(
|
| 110 |
+
this,
|
| 111 |
+
Manifest.permission.ACCESS_FINE_LOCATION,
|
| 112 |
+
) == android.content.pm.PackageManager.PERMISSION_GRANTED
|
| 113 |
+
if (!ok) {
|
| 114 |
+
requestPermissions(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION), 101)
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
private fun requestNotificationPermissionIfNeeded() {
|
| 120 |
+
if (Build.VERSION.SDK_INT < 33) return
|
| 121 |
+
val ok =
|
| 122 |
+
ContextCompat.checkSelfPermission(
|
| 123 |
+
this,
|
| 124 |
+
Manifest.permission.POST_NOTIFICATIONS,
|
| 125 |
+
) == android.content.pm.PackageManager.PERMISSION_GRANTED
|
| 126 |
+
if (!ok) {
|
| 127 |
+
requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), 102)
|
| 128 |
+
}
|
| 129 |
+
}
|
| 130 |
+
}
|
apps/android/app/src/main/java/ai/openclaw/android/MainViewModel.kt
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
import android.app.Application
|
| 4 |
+
import androidx.lifecycle.AndroidViewModel
|
| 5 |
+
import ai.openclaw.android.gateway.GatewayEndpoint
|
| 6 |
+
import ai.openclaw.android.chat.OutgoingAttachment
|
| 7 |
+
import ai.openclaw.android.node.CameraCaptureManager
|
| 8 |
+
import ai.openclaw.android.node.CanvasController
|
| 9 |
+
import ai.openclaw.android.node.ScreenRecordManager
|
| 10 |
+
import ai.openclaw.android.node.SmsManager
|
| 11 |
+
import kotlinx.coroutines.flow.StateFlow
|
| 12 |
+
|
| 13 |
+
class MainViewModel(app: Application) : AndroidViewModel(app) {
|
| 14 |
+
private val runtime: NodeRuntime = (app as NodeApp).runtime
|
| 15 |
+
|
| 16 |
+
val canvas: CanvasController = runtime.canvas
|
| 17 |
+
val camera: CameraCaptureManager = runtime.camera
|
| 18 |
+
val screenRecorder: ScreenRecordManager = runtime.screenRecorder
|
| 19 |
+
val sms: SmsManager = runtime.sms
|
| 20 |
+
|
| 21 |
+
val gateways: StateFlow<List<GatewayEndpoint>> = runtime.gateways
|
| 22 |
+
val discoveryStatusText: StateFlow<String> = runtime.discoveryStatusText
|
| 23 |
+
|
| 24 |
+
val isConnected: StateFlow<Boolean> = runtime.isConnected
|
| 25 |
+
val statusText: StateFlow<String> = runtime.statusText
|
| 26 |
+
val serverName: StateFlow<String?> = runtime.serverName
|
| 27 |
+
val remoteAddress: StateFlow<String?> = runtime.remoteAddress
|
| 28 |
+
val isForeground: StateFlow<Boolean> = runtime.isForeground
|
| 29 |
+
val seamColorArgb: StateFlow<Long> = runtime.seamColorArgb
|
| 30 |
+
val mainSessionKey: StateFlow<String> = runtime.mainSessionKey
|
| 31 |
+
|
| 32 |
+
val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
|
| 33 |
+
val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
|
| 34 |
+
val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
|
| 35 |
+
|
| 36 |
+
val instanceId: StateFlow<String> = runtime.instanceId
|
| 37 |
+
val displayName: StateFlow<String> = runtime.displayName
|
| 38 |
+
val cameraEnabled: StateFlow<Boolean> = runtime.cameraEnabled
|
| 39 |
+
val locationMode: StateFlow<LocationMode> = runtime.locationMode
|
| 40 |
+
val locationPreciseEnabled: StateFlow<Boolean> = runtime.locationPreciseEnabled
|
| 41 |
+
val preventSleep: StateFlow<Boolean> = runtime.preventSleep
|
| 42 |
+
val wakeWords: StateFlow<List<String>> = runtime.wakeWords
|
| 43 |
+
val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
|
| 44 |
+
val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
|
| 45 |
+
val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
|
| 46 |
+
val talkEnabled: StateFlow<Boolean> = runtime.talkEnabled
|
| 47 |
+
val talkStatusText: StateFlow<String> = runtime.talkStatusText
|
| 48 |
+
val talkIsListening: StateFlow<Boolean> = runtime.talkIsListening
|
| 49 |
+
val talkIsSpeaking: StateFlow<Boolean> = runtime.talkIsSpeaking
|
| 50 |
+
val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
|
| 51 |
+
val manualHost: StateFlow<String> = runtime.manualHost
|
| 52 |
+
val manualPort: StateFlow<Int> = runtime.manualPort
|
| 53 |
+
val manualTls: StateFlow<Boolean> = runtime.manualTls
|
| 54 |
+
val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
|
| 55 |
+
|
| 56 |
+
val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
|
| 57 |
+
val chatSessionId: StateFlow<String?> = runtime.chatSessionId
|
| 58 |
+
val chatMessages = runtime.chatMessages
|
| 59 |
+
val chatError: StateFlow<String?> = runtime.chatError
|
| 60 |
+
val chatHealthOk: StateFlow<Boolean> = runtime.chatHealthOk
|
| 61 |
+
val chatThinkingLevel: StateFlow<String> = runtime.chatThinkingLevel
|
| 62 |
+
val chatStreamingAssistantText: StateFlow<String?> = runtime.chatStreamingAssistantText
|
| 63 |
+
val chatPendingToolCalls = runtime.chatPendingToolCalls
|
| 64 |
+
val chatSessions = runtime.chatSessions
|
| 65 |
+
val pendingRunCount: StateFlow<Int> = runtime.pendingRunCount
|
| 66 |
+
|
| 67 |
+
fun setForeground(value: Boolean) {
|
| 68 |
+
runtime.setForeground(value)
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
fun setDisplayName(value: String) {
|
| 72 |
+
runtime.setDisplayName(value)
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
fun setCameraEnabled(value: Boolean) {
|
| 76 |
+
runtime.setCameraEnabled(value)
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
fun setLocationMode(mode: LocationMode) {
|
| 80 |
+
runtime.setLocationMode(mode)
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
fun setLocationPreciseEnabled(value: Boolean) {
|
| 84 |
+
runtime.setLocationPreciseEnabled(value)
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
fun setPreventSleep(value: Boolean) {
|
| 88 |
+
runtime.setPreventSleep(value)
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
fun setManualEnabled(value: Boolean) {
|
| 92 |
+
runtime.setManualEnabled(value)
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
fun setManualHost(value: String) {
|
| 96 |
+
runtime.setManualHost(value)
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
fun setManualPort(value: Int) {
|
| 100 |
+
runtime.setManualPort(value)
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
fun setManualTls(value: Boolean) {
|
| 104 |
+
runtime.setManualTls(value)
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
fun setCanvasDebugStatusEnabled(value: Boolean) {
|
| 108 |
+
runtime.setCanvasDebugStatusEnabled(value)
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
fun setWakeWords(words: List<String>) {
|
| 112 |
+
runtime.setWakeWords(words)
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
fun resetWakeWordsDefaults() {
|
| 116 |
+
runtime.resetWakeWordsDefaults()
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
fun setVoiceWakeMode(mode: VoiceWakeMode) {
|
| 120 |
+
runtime.setVoiceWakeMode(mode)
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
fun setTalkEnabled(enabled: Boolean) {
|
| 124 |
+
runtime.setTalkEnabled(enabled)
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
fun refreshGatewayConnection() {
|
| 128 |
+
runtime.refreshGatewayConnection()
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
fun connect(endpoint: GatewayEndpoint) {
|
| 132 |
+
runtime.connect(endpoint)
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
fun connectManual() {
|
| 136 |
+
runtime.connectManual()
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
fun disconnect() {
|
| 140 |
+
runtime.disconnect()
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
|
| 144 |
+
runtime.handleCanvasA2UIActionFromWebView(payloadJson)
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
fun loadChat(sessionKey: String) {
|
| 148 |
+
runtime.loadChat(sessionKey)
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
fun refreshChat() {
|
| 152 |
+
runtime.refreshChat()
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
fun refreshChatSessions(limit: Int? = null) {
|
| 156 |
+
runtime.refreshChatSessions(limit = limit)
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
fun setChatThinkingLevel(level: String) {
|
| 160 |
+
runtime.setChatThinkingLevel(level)
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
fun switchChatSession(sessionKey: String) {
|
| 164 |
+
runtime.switchChatSession(sessionKey)
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
fun abortChat() {
|
| 168 |
+
runtime.abortChat()
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
|
| 172 |
+
runtime.sendChat(message = message, thinking = thinking, attachments = attachments)
|
| 173 |
+
}
|
| 174 |
+
}
|
apps/android/app/src/main/java/ai/openclaw/android/NodeApp.kt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
import android.app.Application
|
| 4 |
+
import android.os.StrictMode
|
| 5 |
+
|
| 6 |
+
class NodeApp : Application() {
|
| 7 |
+
val runtime: NodeRuntime by lazy { NodeRuntime(this) }
|
| 8 |
+
|
| 9 |
+
override fun onCreate() {
|
| 10 |
+
super.onCreate()
|
| 11 |
+
if (BuildConfig.DEBUG) {
|
| 12 |
+
StrictMode.setThreadPolicy(
|
| 13 |
+
StrictMode.ThreadPolicy.Builder()
|
| 14 |
+
.detectAll()
|
| 15 |
+
.penaltyLog()
|
| 16 |
+
.build(),
|
| 17 |
+
)
|
| 18 |
+
StrictMode.setVmPolicy(
|
| 19 |
+
StrictMode.VmPolicy.Builder()
|
| 20 |
+
.detectAll()
|
| 21 |
+
.penaltyLog()
|
| 22 |
+
.build(),
|
| 23 |
+
)
|
| 24 |
+
}
|
| 25 |
+
}
|
| 26 |
+
}
|
apps/android/app/src/main/java/ai/openclaw/android/NodeForegroundService.kt
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
import android.app.Notification
|
| 4 |
+
import android.app.NotificationChannel
|
| 5 |
+
import android.app.NotificationManager
|
| 6 |
+
import android.app.Service
|
| 7 |
+
import android.app.PendingIntent
|
| 8 |
+
import android.Manifest
|
| 9 |
+
import android.content.Context
|
| 10 |
+
import android.content.Intent
|
| 11 |
+
import android.content.pm.PackageManager
|
| 12 |
+
import android.content.pm.ServiceInfo
|
| 13 |
+
import androidx.core.app.NotificationCompat
|
| 14 |
+
import androidx.core.content.ContextCompat
|
| 15 |
+
import kotlinx.coroutines.CoroutineScope
|
| 16 |
+
import kotlinx.coroutines.Dispatchers
|
| 17 |
+
import kotlinx.coroutines.Job
|
| 18 |
+
import kotlinx.coroutines.SupervisorJob
|
| 19 |
+
import kotlinx.coroutines.cancel
|
| 20 |
+
import kotlinx.coroutines.flow.combine
|
| 21 |
+
import kotlinx.coroutines.launch
|
| 22 |
+
|
| 23 |
+
class NodeForegroundService : Service() {
|
| 24 |
+
private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
|
| 25 |
+
private var notificationJob: Job? = null
|
| 26 |
+
private var lastRequiresMic = false
|
| 27 |
+
private var didStartForeground = false
|
| 28 |
+
|
| 29 |
+
override fun onCreate() {
|
| 30 |
+
super.onCreate()
|
| 31 |
+
ensureChannel()
|
| 32 |
+
val initial = buildNotification(title = "OpenClaw Node", text = "Starting…")
|
| 33 |
+
startForegroundWithTypes(notification = initial, requiresMic = false)
|
| 34 |
+
|
| 35 |
+
val runtime = (application as NodeApp).runtime
|
| 36 |
+
notificationJob =
|
| 37 |
+
scope.launch {
|
| 38 |
+
combine(
|
| 39 |
+
runtime.statusText,
|
| 40 |
+
runtime.serverName,
|
| 41 |
+
runtime.isConnected,
|
| 42 |
+
runtime.voiceWakeMode,
|
| 43 |
+
runtime.voiceWakeIsListening,
|
| 44 |
+
) { status, server, connected, voiceMode, voiceListening ->
|
| 45 |
+
Quint(status, server, connected, voiceMode, voiceListening)
|
| 46 |
+
}.collect { (status, server, connected, voiceMode, voiceListening) ->
|
| 47 |
+
val title = if (connected) "OpenClaw Node · Connected" else "OpenClaw Node"
|
| 48 |
+
val voiceSuffix =
|
| 49 |
+
if (voiceMode == VoiceWakeMode.Always) {
|
| 50 |
+
if (voiceListening) " · Voice Wake: Listening" else " · Voice Wake: Paused"
|
| 51 |
+
} else {
|
| 52 |
+
""
|
| 53 |
+
}
|
| 54 |
+
val text = (server?.let { "$status · $it" } ?: status) + voiceSuffix
|
| 55 |
+
|
| 56 |
+
val requiresMic =
|
| 57 |
+
voiceMode == VoiceWakeMode.Always && hasRecordAudioPermission()
|
| 58 |
+
startForegroundWithTypes(
|
| 59 |
+
notification = buildNotification(title = title, text = text),
|
| 60 |
+
requiresMic = requiresMic,
|
| 61 |
+
)
|
| 62 |
+
}
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
| 67 |
+
when (intent?.action) {
|
| 68 |
+
ACTION_STOP -> {
|
| 69 |
+
(application as NodeApp).runtime.disconnect()
|
| 70 |
+
stopSelf()
|
| 71 |
+
return START_NOT_STICKY
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
// Keep running; connection is managed by NodeRuntime (auto-reconnect + manual).
|
| 75 |
+
return START_STICKY
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
override fun onDestroy() {
|
| 79 |
+
notificationJob?.cancel()
|
| 80 |
+
scope.cancel()
|
| 81 |
+
super.onDestroy()
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
override fun onBind(intent: Intent?) = null
|
| 85 |
+
|
| 86 |
+
private fun ensureChannel() {
|
| 87 |
+
val mgr = getSystemService(NotificationManager::class.java)
|
| 88 |
+
val channel =
|
| 89 |
+
NotificationChannel(
|
| 90 |
+
CHANNEL_ID,
|
| 91 |
+
"Connection",
|
| 92 |
+
NotificationManager.IMPORTANCE_LOW,
|
| 93 |
+
).apply {
|
| 94 |
+
description = "OpenClaw node connection status"
|
| 95 |
+
setShowBadge(false)
|
| 96 |
+
}
|
| 97 |
+
mgr.createNotificationChannel(channel)
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
private fun buildNotification(title: String, text: String): Notification {
|
| 101 |
+
val launchIntent = Intent(this, MainActivity::class.java).apply {
|
| 102 |
+
flags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP
|
| 103 |
+
}
|
| 104 |
+
val launchPending =
|
| 105 |
+
PendingIntent.getActivity(
|
| 106 |
+
this,
|
| 107 |
+
1,
|
| 108 |
+
launchIntent,
|
| 109 |
+
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP)
|
| 113 |
+
val stopPending =
|
| 114 |
+
PendingIntent.getService(
|
| 115 |
+
this,
|
| 116 |
+
2,
|
| 117 |
+
stopIntent,
|
| 118 |
+
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
return NotificationCompat.Builder(this, CHANNEL_ID)
|
| 122 |
+
.setSmallIcon(R.mipmap.ic_launcher)
|
| 123 |
+
.setContentTitle(title)
|
| 124 |
+
.setContentText(text)
|
| 125 |
+
.setContentIntent(launchPending)
|
| 126 |
+
.setOngoing(true)
|
| 127 |
+
.setOnlyAlertOnce(true)
|
| 128 |
+
.setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE)
|
| 129 |
+
.addAction(0, "Disconnect", stopPending)
|
| 130 |
+
.build()
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
private fun updateNotification(notification: Notification) {
|
| 134 |
+
val mgr = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
|
| 135 |
+
mgr.notify(NOTIFICATION_ID, notification)
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
|
| 139 |
+
if (didStartForeground && requiresMic == lastRequiresMic) {
|
| 140 |
+
updateNotification(notification)
|
| 141 |
+
return
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
lastRequiresMic = requiresMic
|
| 145 |
+
val types =
|
| 146 |
+
if (requiresMic) {
|
| 147 |
+
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
| 148 |
+
} else {
|
| 149 |
+
ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
|
| 150 |
+
}
|
| 151 |
+
startForeground(NOTIFICATION_ID, notification, types)
|
| 152 |
+
didStartForeground = true
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
private fun hasRecordAudioPermission(): Boolean {
|
| 156 |
+
return (
|
| 157 |
+
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
|
| 158 |
+
PackageManager.PERMISSION_GRANTED
|
| 159 |
+
)
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
companion object {
|
| 163 |
+
private const val CHANNEL_ID = "connection"
|
| 164 |
+
private const val NOTIFICATION_ID = 1
|
| 165 |
+
|
| 166 |
+
private const val ACTION_STOP = "ai.openclaw.android.action.STOP"
|
| 167 |
+
|
| 168 |
+
fun start(context: Context) {
|
| 169 |
+
val intent = Intent(context, NodeForegroundService::class.java)
|
| 170 |
+
context.startForegroundService(intent)
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
fun stop(context: Context) {
|
| 174 |
+
val intent = Intent(context, NodeForegroundService::class.java).setAction(ACTION_STOP)
|
| 175 |
+
context.startService(intent)
|
| 176 |
+
}
|
| 177 |
+
}
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
private data class Quint<A, B, C, D, E>(val first: A, val second: B, val third: C, val fourth: D, val fifth: E)
|
apps/android/app/src/main/java/ai/openclaw/android/NodeRuntime.kt
ADDED
|
@@ -0,0 +1,1271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
import android.Manifest
|
| 4 |
+
import android.content.Context
|
| 5 |
+
import android.content.pm.PackageManager
|
| 6 |
+
import android.location.LocationManager
|
| 7 |
+
import android.os.Build
|
| 8 |
+
import android.os.SystemClock
|
| 9 |
+
import androidx.core.content.ContextCompat
|
| 10 |
+
import ai.openclaw.android.chat.ChatController
|
| 11 |
+
import ai.openclaw.android.chat.ChatMessage
|
| 12 |
+
import ai.openclaw.android.chat.ChatPendingToolCall
|
| 13 |
+
import ai.openclaw.android.chat.ChatSessionEntry
|
| 14 |
+
import ai.openclaw.android.chat.OutgoingAttachment
|
| 15 |
+
import ai.openclaw.android.gateway.DeviceAuthStore
|
| 16 |
+
import ai.openclaw.android.gateway.DeviceIdentityStore
|
| 17 |
+
import ai.openclaw.android.gateway.GatewayClientInfo
|
| 18 |
+
import ai.openclaw.android.gateway.GatewayConnectOptions
|
| 19 |
+
import ai.openclaw.android.gateway.GatewayDiscovery
|
| 20 |
+
import ai.openclaw.android.gateway.GatewayEndpoint
|
| 21 |
+
import ai.openclaw.android.gateway.GatewaySession
|
| 22 |
+
import ai.openclaw.android.gateway.GatewayTlsParams
|
| 23 |
+
import ai.openclaw.android.node.CameraCaptureManager
|
| 24 |
+
import ai.openclaw.android.node.LocationCaptureManager
|
| 25 |
+
import ai.openclaw.android.BuildConfig
|
| 26 |
+
import ai.openclaw.android.node.CanvasController
|
| 27 |
+
import ai.openclaw.android.node.ScreenRecordManager
|
| 28 |
+
import ai.openclaw.android.node.SmsManager
|
| 29 |
+
import ai.openclaw.android.protocol.OpenClawCapability
|
| 30 |
+
import ai.openclaw.android.protocol.OpenClawCameraCommand
|
| 31 |
+
import ai.openclaw.android.protocol.OpenClawCanvasA2UIAction
|
| 32 |
+
import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
|
| 33 |
+
import ai.openclaw.android.protocol.OpenClawCanvasCommand
|
| 34 |
+
import ai.openclaw.android.protocol.OpenClawScreenCommand
|
| 35 |
+
import ai.openclaw.android.protocol.OpenClawLocationCommand
|
| 36 |
+
import ai.openclaw.android.protocol.OpenClawSmsCommand
|
| 37 |
+
import ai.openclaw.android.voice.TalkModeManager
|
| 38 |
+
import ai.openclaw.android.voice.VoiceWakeManager
|
| 39 |
+
import kotlinx.coroutines.CoroutineScope
|
| 40 |
+
import kotlinx.coroutines.Dispatchers
|
| 41 |
+
import kotlinx.coroutines.Job
|
| 42 |
+
import kotlinx.coroutines.SupervisorJob
|
| 43 |
+
import kotlinx.coroutines.TimeoutCancellationException
|
| 44 |
+
import kotlinx.coroutines.delay
|
| 45 |
+
import kotlinx.coroutines.flow.MutableStateFlow
|
| 46 |
+
import kotlinx.coroutines.flow.StateFlow
|
| 47 |
+
import kotlinx.coroutines.flow.asStateFlow
|
| 48 |
+
import kotlinx.coroutines.flow.combine
|
| 49 |
+
import kotlinx.coroutines.flow.collect
|
| 50 |
+
import kotlinx.coroutines.flow.distinctUntilChanged
|
| 51 |
+
import kotlinx.coroutines.launch
|
| 52 |
+
import kotlinx.serialization.json.Json
|
| 53 |
+
import kotlinx.serialization.json.JsonArray
|
| 54 |
+
import kotlinx.serialization.json.JsonElement
|
| 55 |
+
import kotlinx.serialization.json.JsonNull
|
| 56 |
+
import kotlinx.serialization.json.JsonObject
|
| 57 |
+
import kotlinx.serialization.json.JsonPrimitive
|
| 58 |
+
import kotlinx.serialization.json.buildJsonObject
|
| 59 |
+
import java.util.concurrent.atomic.AtomicLong
|
| 60 |
+
|
| 61 |
+
class NodeRuntime(context: Context) {
|
| 62 |
+
private val appContext = context.applicationContext
|
| 63 |
+
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
| 64 |
+
|
| 65 |
+
val prefs = SecurePrefs(appContext)
|
| 66 |
+
private val deviceAuthStore = DeviceAuthStore(prefs)
|
| 67 |
+
val canvas = CanvasController()
|
| 68 |
+
val camera = CameraCaptureManager(appContext)
|
| 69 |
+
val location = LocationCaptureManager(appContext)
|
| 70 |
+
val screenRecorder = ScreenRecordManager(appContext)
|
| 71 |
+
val sms = SmsManager(appContext)
|
| 72 |
+
private val json = Json { ignoreUnknownKeys = true }
|
| 73 |
+
|
| 74 |
+
private val externalAudioCaptureActive = MutableStateFlow(false)
|
| 75 |
+
|
| 76 |
+
private val voiceWake: VoiceWakeManager by lazy {
|
| 77 |
+
VoiceWakeManager(
|
| 78 |
+
context = appContext,
|
| 79 |
+
scope = scope,
|
| 80 |
+
onCommand = { command ->
|
| 81 |
+
nodeSession.sendNodeEvent(
|
| 82 |
+
event = "agent.request",
|
| 83 |
+
payloadJson =
|
| 84 |
+
buildJsonObject {
|
| 85 |
+
put("message", JsonPrimitive(command))
|
| 86 |
+
put("sessionKey", JsonPrimitive(resolveMainSessionKey()))
|
| 87 |
+
put("thinking", JsonPrimitive(chatThinkingLevel.value))
|
| 88 |
+
put("deliver", JsonPrimitive(false))
|
| 89 |
+
}.toString(),
|
| 90 |
+
)
|
| 91 |
+
},
|
| 92 |
+
)
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
val voiceWakeIsListening: StateFlow<Boolean>
|
| 96 |
+
get() = voiceWake.isListening
|
| 97 |
+
|
| 98 |
+
val voiceWakeStatusText: StateFlow<String>
|
| 99 |
+
get() = voiceWake.statusText
|
| 100 |
+
|
| 101 |
+
val talkStatusText: StateFlow<String>
|
| 102 |
+
get() = talkMode.statusText
|
| 103 |
+
|
| 104 |
+
val talkIsListening: StateFlow<Boolean>
|
| 105 |
+
get() = talkMode.isListening
|
| 106 |
+
|
| 107 |
+
val talkIsSpeaking: StateFlow<Boolean>
|
| 108 |
+
get() = talkMode.isSpeaking
|
| 109 |
+
|
| 110 |
+
private val discovery = GatewayDiscovery(appContext, scope = scope)
|
| 111 |
+
val gateways: StateFlow<List<GatewayEndpoint>> = discovery.gateways
|
| 112 |
+
val discoveryStatusText: StateFlow<String> = discovery.statusText
|
| 113 |
+
|
| 114 |
+
private val identityStore = DeviceIdentityStore(appContext)
|
| 115 |
+
|
| 116 |
+
private val _isConnected = MutableStateFlow(false)
|
| 117 |
+
val isConnected: StateFlow<Boolean> = _isConnected.asStateFlow()
|
| 118 |
+
|
| 119 |
+
private val _statusText = MutableStateFlow("Offline")
|
| 120 |
+
val statusText: StateFlow<String> = _statusText.asStateFlow()
|
| 121 |
+
|
| 122 |
+
private val _mainSessionKey = MutableStateFlow("main")
|
| 123 |
+
val mainSessionKey: StateFlow<String> = _mainSessionKey.asStateFlow()
|
| 124 |
+
|
| 125 |
+
private val cameraHudSeq = AtomicLong(0)
|
| 126 |
+
private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
|
| 127 |
+
val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
|
| 128 |
+
|
| 129 |
+
private val _cameraFlashToken = MutableStateFlow(0L)
|
| 130 |
+
val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
|
| 131 |
+
|
| 132 |
+
private val _screenRecordActive = MutableStateFlow(false)
|
| 133 |
+
val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
|
| 134 |
+
|
| 135 |
+
private val _serverName = MutableStateFlow<String?>(null)
|
| 136 |
+
val serverName: StateFlow<String?> = _serverName.asStateFlow()
|
| 137 |
+
|
| 138 |
+
private val _remoteAddress = MutableStateFlow<String?>(null)
|
| 139 |
+
val remoteAddress: StateFlow<String?> = _remoteAddress.asStateFlow()
|
| 140 |
+
|
| 141 |
+
private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB)
|
| 142 |
+
val seamColorArgb: StateFlow<Long> = _seamColorArgb.asStateFlow()
|
| 143 |
+
|
| 144 |
+
private val _isForeground = MutableStateFlow(true)
|
| 145 |
+
val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
|
| 146 |
+
|
| 147 |
+
private var lastAutoA2uiUrl: String? = null
|
| 148 |
+
private var operatorConnected = false
|
| 149 |
+
private var nodeConnected = false
|
| 150 |
+
private var operatorStatusText: String = "Offline"
|
| 151 |
+
private var nodeStatusText: String = "Offline"
|
| 152 |
+
private var connectedEndpoint: GatewayEndpoint? = null
|
| 153 |
+
|
| 154 |
+
private val operatorSession =
|
| 155 |
+
GatewaySession(
|
| 156 |
+
scope = scope,
|
| 157 |
+
identityStore = identityStore,
|
| 158 |
+
deviceAuthStore = deviceAuthStore,
|
| 159 |
+
onConnected = { name, remote, mainSessionKey ->
|
| 160 |
+
operatorConnected = true
|
| 161 |
+
operatorStatusText = "Connected"
|
| 162 |
+
_serverName.value = name
|
| 163 |
+
_remoteAddress.value = remote
|
| 164 |
+
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
|
| 165 |
+
applyMainSessionKey(mainSessionKey)
|
| 166 |
+
updateStatus()
|
| 167 |
+
scope.launch { refreshBrandingFromGateway() }
|
| 168 |
+
scope.launch { refreshWakeWordsFromGateway() }
|
| 169 |
+
},
|
| 170 |
+
onDisconnected = { message ->
|
| 171 |
+
operatorConnected = false
|
| 172 |
+
operatorStatusText = message
|
| 173 |
+
_serverName.value = null
|
| 174 |
+
_remoteAddress.value = null
|
| 175 |
+
_seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
|
| 176 |
+
if (!isCanonicalMainSessionKey(_mainSessionKey.value)) {
|
| 177 |
+
_mainSessionKey.value = "main"
|
| 178 |
+
}
|
| 179 |
+
val mainKey = resolveMainSessionKey()
|
| 180 |
+
talkMode.setMainSessionKey(mainKey)
|
| 181 |
+
chat.applyMainSessionKey(mainKey)
|
| 182 |
+
chat.onDisconnected(message)
|
| 183 |
+
updateStatus()
|
| 184 |
+
},
|
| 185 |
+
onEvent = { event, payloadJson ->
|
| 186 |
+
handleGatewayEvent(event, payloadJson)
|
| 187 |
+
},
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
private val nodeSession =
|
| 191 |
+
GatewaySession(
|
| 192 |
+
scope = scope,
|
| 193 |
+
identityStore = identityStore,
|
| 194 |
+
deviceAuthStore = deviceAuthStore,
|
| 195 |
+
onConnected = { _, _, _ ->
|
| 196 |
+
nodeConnected = true
|
| 197 |
+
nodeStatusText = "Connected"
|
| 198 |
+
updateStatus()
|
| 199 |
+
maybeNavigateToA2uiOnConnect()
|
| 200 |
+
},
|
| 201 |
+
onDisconnected = { message ->
|
| 202 |
+
nodeConnected = false
|
| 203 |
+
nodeStatusText = message
|
| 204 |
+
updateStatus()
|
| 205 |
+
showLocalCanvasOnDisconnect()
|
| 206 |
+
},
|
| 207 |
+
onEvent = { _, _ -> },
|
| 208 |
+
onInvoke = { req ->
|
| 209 |
+
handleInvoke(req.command, req.paramsJson)
|
| 210 |
+
},
|
| 211 |
+
onTlsFingerprint = { stableId, fingerprint ->
|
| 212 |
+
prefs.saveGatewayTlsFingerprint(stableId, fingerprint)
|
| 213 |
+
},
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
private val chat: ChatController =
|
| 217 |
+
ChatController(
|
| 218 |
+
scope = scope,
|
| 219 |
+
session = operatorSession,
|
| 220 |
+
json = json,
|
| 221 |
+
supportsChatSubscribe = false,
|
| 222 |
+
)
|
| 223 |
+
private val talkMode: TalkModeManager by lazy {
|
| 224 |
+
TalkModeManager(
|
| 225 |
+
context = appContext,
|
| 226 |
+
scope = scope,
|
| 227 |
+
session = operatorSession,
|
| 228 |
+
supportsChatSubscribe = false,
|
| 229 |
+
isConnected = { operatorConnected },
|
| 230 |
+
)
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
private fun applyMainSessionKey(candidate: String?) {
|
| 234 |
+
val trimmed = candidate?.trim().orEmpty()
|
| 235 |
+
if (trimmed.isEmpty()) return
|
| 236 |
+
if (isCanonicalMainSessionKey(_mainSessionKey.value)) return
|
| 237 |
+
if (_mainSessionKey.value == trimmed) return
|
| 238 |
+
_mainSessionKey.value = trimmed
|
| 239 |
+
talkMode.setMainSessionKey(trimmed)
|
| 240 |
+
chat.applyMainSessionKey(trimmed)
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
private fun updateStatus() {
|
| 244 |
+
_isConnected.value = operatorConnected
|
| 245 |
+
_statusText.value =
|
| 246 |
+
when {
|
| 247 |
+
operatorConnected && nodeConnected -> "Connected"
|
| 248 |
+
operatorConnected && !nodeConnected -> "Connected (node offline)"
|
| 249 |
+
!operatorConnected && nodeConnected -> "Connected (operator offline)"
|
| 250 |
+
operatorStatusText.isNotBlank() && operatorStatusText != "Offline" -> operatorStatusText
|
| 251 |
+
else -> nodeStatusText
|
| 252 |
+
}
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
private fun resolveMainSessionKey(): String {
|
| 256 |
+
val trimmed = _mainSessionKey.value.trim()
|
| 257 |
+
return if (trimmed.isEmpty()) "main" else trimmed
|
| 258 |
+
}
|
| 259 |
+
|
| 260 |
+
private fun maybeNavigateToA2uiOnConnect() {
|
| 261 |
+
val a2uiUrl = resolveA2uiHostUrl() ?: return
|
| 262 |
+
val current = canvas.currentUrl()?.trim().orEmpty()
|
| 263 |
+
if (current.isEmpty() || current == lastAutoA2uiUrl) {
|
| 264 |
+
lastAutoA2uiUrl = a2uiUrl
|
| 265 |
+
canvas.navigate(a2uiUrl)
|
| 266 |
+
}
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
private fun showLocalCanvasOnDisconnect() {
|
| 270 |
+
lastAutoA2uiUrl = null
|
| 271 |
+
canvas.navigate("")
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
val instanceId: StateFlow<String> = prefs.instanceId
|
| 275 |
+
val displayName: StateFlow<String> = prefs.displayName
|
| 276 |
+
val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled
|
| 277 |
+
val locationMode: StateFlow<LocationMode> = prefs.locationMode
|
| 278 |
+
val locationPreciseEnabled: StateFlow<Boolean> = prefs.locationPreciseEnabled
|
| 279 |
+
val preventSleep: StateFlow<Boolean> = prefs.preventSleep
|
| 280 |
+
val wakeWords: StateFlow<List<String>> = prefs.wakeWords
|
| 281 |
+
val voiceWakeMode: StateFlow<VoiceWakeMode> = prefs.voiceWakeMode
|
| 282 |
+
val talkEnabled: StateFlow<Boolean> = prefs.talkEnabled
|
| 283 |
+
val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
|
| 284 |
+
val manualHost: StateFlow<String> = prefs.manualHost
|
| 285 |
+
val manualPort: StateFlow<Int> = prefs.manualPort
|
| 286 |
+
val manualTls: StateFlow<Boolean> = prefs.manualTls
|
| 287 |
+
val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
|
| 288 |
+
val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
|
| 289 |
+
|
| 290 |
+
private var didAutoConnect = false
|
| 291 |
+
private var suppressWakeWordsSync = false
|
| 292 |
+
private var wakeWordsSyncJob: Job? = null
|
| 293 |
+
|
| 294 |
+
val chatSessionKey: StateFlow<String> = chat.sessionKey
|
| 295 |
+
val chatSessionId: StateFlow<String?> = chat.sessionId
|
| 296 |
+
val chatMessages: StateFlow<List<ChatMessage>> = chat.messages
|
| 297 |
+
val chatError: StateFlow<String?> = chat.errorText
|
| 298 |
+
val chatHealthOk: StateFlow<Boolean> = chat.healthOk
|
| 299 |
+
val chatThinkingLevel: StateFlow<String> = chat.thinkingLevel
|
| 300 |
+
val chatStreamingAssistantText: StateFlow<String?> = chat.streamingAssistantText
|
| 301 |
+
val chatPendingToolCalls: StateFlow<List<ChatPendingToolCall>> = chat.pendingToolCalls
|
| 302 |
+
val chatSessions: StateFlow<List<ChatSessionEntry>> = chat.sessions
|
| 303 |
+
val pendingRunCount: StateFlow<Int> = chat.pendingRunCount
|
| 304 |
+
|
| 305 |
+
init {
|
| 306 |
+
scope.launch {
|
| 307 |
+
combine(
|
| 308 |
+
voiceWakeMode,
|
| 309 |
+
isForeground,
|
| 310 |
+
externalAudioCaptureActive,
|
| 311 |
+
wakeWords,
|
| 312 |
+
) { mode, foreground, externalAudio, words ->
|
| 313 |
+
Quad(mode, foreground, externalAudio, words)
|
| 314 |
+
}.distinctUntilChanged()
|
| 315 |
+
.collect { (mode, foreground, externalAudio, words) ->
|
| 316 |
+
voiceWake.setTriggerWords(words)
|
| 317 |
+
|
| 318 |
+
val shouldListen =
|
| 319 |
+
when (mode) {
|
| 320 |
+
VoiceWakeMode.Off -> false
|
| 321 |
+
VoiceWakeMode.Foreground -> foreground
|
| 322 |
+
VoiceWakeMode.Always -> true
|
| 323 |
+
} && !externalAudio
|
| 324 |
+
|
| 325 |
+
if (!shouldListen) {
|
| 326 |
+
voiceWake.stop(statusText = if (mode == VoiceWakeMode.Off) "Off" else "Paused")
|
| 327 |
+
return@collect
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
if (!hasRecordAudioPermission()) {
|
| 331 |
+
voiceWake.stop(statusText = "Microphone permission required")
|
| 332 |
+
return@collect
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
voiceWake.start()
|
| 336 |
+
}
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
scope.launch {
|
| 340 |
+
talkEnabled.collect { enabled ->
|
| 341 |
+
talkMode.setEnabled(enabled)
|
| 342 |
+
externalAudioCaptureActive.value = enabled
|
| 343 |
+
}
|
| 344 |
+
}
|
| 345 |
+
|
| 346 |
+
scope.launch(Dispatchers.Default) {
|
| 347 |
+
gateways.collect { list ->
|
| 348 |
+
if (list.isNotEmpty()) {
|
| 349 |
+
// Persist the last discovered gateway (best-effort UX parity with iOS).
|
| 350 |
+
prefs.setLastDiscoveredStableId(list.last().stableId)
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
if (didAutoConnect) return@collect
|
| 354 |
+
if (_isConnected.value) return@collect
|
| 355 |
+
|
| 356 |
+
if (manualEnabled.value) {
|
| 357 |
+
val host = manualHost.value.trim()
|
| 358 |
+
val port = manualPort.value
|
| 359 |
+
if (host.isNotEmpty() && port in 1..65535) {
|
| 360 |
+
didAutoConnect = true
|
| 361 |
+
connect(GatewayEndpoint.manual(host = host, port = port))
|
| 362 |
+
}
|
| 363 |
+
return@collect
|
| 364 |
+
}
|
| 365 |
+
|
| 366 |
+
val targetStableId = lastDiscoveredStableId.value.trim()
|
| 367 |
+
if (targetStableId.isEmpty()) return@collect
|
| 368 |
+
val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect
|
| 369 |
+
didAutoConnect = true
|
| 370 |
+
connect(target)
|
| 371 |
+
}
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
scope.launch {
|
| 375 |
+
combine(
|
| 376 |
+
canvasDebugStatusEnabled,
|
| 377 |
+
statusText,
|
| 378 |
+
serverName,
|
| 379 |
+
remoteAddress,
|
| 380 |
+
) { debugEnabled, status, server, remote ->
|
| 381 |
+
Quad(debugEnabled, status, server, remote)
|
| 382 |
+
}.distinctUntilChanged()
|
| 383 |
+
.collect { (debugEnabled, status, server, remote) ->
|
| 384 |
+
canvas.setDebugStatusEnabled(debugEnabled)
|
| 385 |
+
if (!debugEnabled) return@collect
|
| 386 |
+
canvas.setDebugStatus(status, server ?: remote)
|
| 387 |
+
}
|
| 388 |
+
}
|
| 389 |
+
}
|
| 390 |
+
|
| 391 |
+
fun setForeground(value: Boolean) {
|
| 392 |
+
_isForeground.value = value
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
fun setDisplayName(value: String) {
|
| 396 |
+
prefs.setDisplayName(value)
|
| 397 |
+
}
|
| 398 |
+
|
| 399 |
+
fun setCameraEnabled(value: Boolean) {
|
| 400 |
+
prefs.setCameraEnabled(value)
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
fun setLocationMode(mode: LocationMode) {
|
| 404 |
+
prefs.setLocationMode(mode)
|
| 405 |
+
}
|
| 406 |
+
|
| 407 |
+
fun setLocationPreciseEnabled(value: Boolean) {
|
| 408 |
+
prefs.setLocationPreciseEnabled(value)
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
fun setPreventSleep(value: Boolean) {
|
| 412 |
+
prefs.setPreventSleep(value)
|
| 413 |
+
}
|
| 414 |
+
|
| 415 |
+
fun setManualEnabled(value: Boolean) {
|
| 416 |
+
prefs.setManualEnabled(value)
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
fun setManualHost(value: String) {
|
| 420 |
+
prefs.setManualHost(value)
|
| 421 |
+
}
|
| 422 |
+
|
| 423 |
+
fun setManualPort(value: Int) {
|
| 424 |
+
prefs.setManualPort(value)
|
| 425 |
+
}
|
| 426 |
+
|
| 427 |
+
fun setManualTls(value: Boolean) {
|
| 428 |
+
prefs.setManualTls(value)
|
| 429 |
+
}
|
| 430 |
+
|
| 431 |
+
fun setCanvasDebugStatusEnabled(value: Boolean) {
|
| 432 |
+
prefs.setCanvasDebugStatusEnabled(value)
|
| 433 |
+
}
|
| 434 |
+
|
| 435 |
+
fun setWakeWords(words: List<String>) {
|
| 436 |
+
prefs.setWakeWords(words)
|
| 437 |
+
scheduleWakeWordsSyncIfNeeded()
|
| 438 |
+
}
|
| 439 |
+
|
| 440 |
+
fun resetWakeWordsDefaults() {
|
| 441 |
+
setWakeWords(SecurePrefs.defaultWakeWords)
|
| 442 |
+
}
|
| 443 |
+
|
| 444 |
+
fun setVoiceWakeMode(mode: VoiceWakeMode) {
|
| 445 |
+
prefs.setVoiceWakeMode(mode)
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
+
fun setTalkEnabled(value: Boolean) {
|
| 449 |
+
prefs.setTalkEnabled(value)
|
| 450 |
+
}
|
| 451 |
+
|
| 452 |
+
private fun buildInvokeCommands(): List<String> =
|
| 453 |
+
buildList {
|
| 454 |
+
add(OpenClawCanvasCommand.Present.rawValue)
|
| 455 |
+
add(OpenClawCanvasCommand.Hide.rawValue)
|
| 456 |
+
add(OpenClawCanvasCommand.Navigate.rawValue)
|
| 457 |
+
add(OpenClawCanvasCommand.Eval.rawValue)
|
| 458 |
+
add(OpenClawCanvasCommand.Snapshot.rawValue)
|
| 459 |
+
add(OpenClawCanvasA2UICommand.Push.rawValue)
|
| 460 |
+
add(OpenClawCanvasA2UICommand.PushJSONL.rawValue)
|
| 461 |
+
add(OpenClawCanvasA2UICommand.Reset.rawValue)
|
| 462 |
+
add(OpenClawScreenCommand.Record.rawValue)
|
| 463 |
+
if (cameraEnabled.value) {
|
| 464 |
+
add(OpenClawCameraCommand.Snap.rawValue)
|
| 465 |
+
add(OpenClawCameraCommand.Clip.rawValue)
|
| 466 |
+
}
|
| 467 |
+
if (locationMode.value != LocationMode.Off) {
|
| 468 |
+
add(OpenClawLocationCommand.Get.rawValue)
|
| 469 |
+
}
|
| 470 |
+
if (sms.canSendSms()) {
|
| 471 |
+
add(OpenClawSmsCommand.Send.rawValue)
|
| 472 |
+
}
|
| 473 |
+
}
|
| 474 |
+
|
| 475 |
+
private fun buildCapabilities(): List<String> =
|
| 476 |
+
buildList {
|
| 477 |
+
add(OpenClawCapability.Canvas.rawValue)
|
| 478 |
+
add(OpenClawCapability.Screen.rawValue)
|
| 479 |
+
if (cameraEnabled.value) add(OpenClawCapability.Camera.rawValue)
|
| 480 |
+
if (sms.canSendSms()) add(OpenClawCapability.Sms.rawValue)
|
| 481 |
+
if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
|
| 482 |
+
add(OpenClawCapability.VoiceWake.rawValue)
|
| 483 |
+
}
|
| 484 |
+
if (locationMode.value != LocationMode.Off) {
|
| 485 |
+
add(OpenClawCapability.Location.rawValue)
|
| 486 |
+
}
|
| 487 |
+
}
|
| 488 |
+
|
| 489 |
+
private fun resolvedVersionName(): String {
|
| 490 |
+
val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
|
| 491 |
+
return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
|
| 492 |
+
"$versionName-dev"
|
| 493 |
+
} else {
|
| 494 |
+
versionName
|
| 495 |
+
}
|
| 496 |
+
}
|
| 497 |
+
|
| 498 |
+
private fun resolveModelIdentifier(): String? {
|
| 499 |
+
return listOfNotNull(Build.MANUFACTURER, Build.MODEL)
|
| 500 |
+
.joinToString(" ")
|
| 501 |
+
.trim()
|
| 502 |
+
.ifEmpty { null }
|
| 503 |
+
}
|
| 504 |
+
|
| 505 |
+
private fun buildUserAgent(): String {
|
| 506 |
+
val version = resolvedVersionName()
|
| 507 |
+
val release = Build.VERSION.RELEASE?.trim().orEmpty()
|
| 508 |
+
val releaseLabel = if (release.isEmpty()) "unknown" else release
|
| 509 |
+
return "OpenClawAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})"
|
| 510 |
+
}
|
| 511 |
+
|
| 512 |
+
private fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo {
|
| 513 |
+
return GatewayClientInfo(
|
| 514 |
+
id = clientId,
|
| 515 |
+
displayName = displayName.value,
|
| 516 |
+
version = resolvedVersionName(),
|
| 517 |
+
platform = "android",
|
| 518 |
+
mode = clientMode,
|
| 519 |
+
instanceId = instanceId.value,
|
| 520 |
+
deviceFamily = "Android",
|
| 521 |
+
modelIdentifier = resolveModelIdentifier(),
|
| 522 |
+
)
|
| 523 |
+
}
|
| 524 |
+
|
| 525 |
+
private fun buildNodeConnectOptions(): GatewayConnectOptions {
|
| 526 |
+
return GatewayConnectOptions(
|
| 527 |
+
role = "node",
|
| 528 |
+
scopes = emptyList(),
|
| 529 |
+
caps = buildCapabilities(),
|
| 530 |
+
commands = buildInvokeCommands(),
|
| 531 |
+
permissions = emptyMap(),
|
| 532 |
+
client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"),
|
| 533 |
+
userAgent = buildUserAgent(),
|
| 534 |
+
)
|
| 535 |
+
}
|
| 536 |
+
|
| 537 |
+
private fun buildOperatorConnectOptions(): GatewayConnectOptions {
|
| 538 |
+
return GatewayConnectOptions(
|
| 539 |
+
role = "operator",
|
| 540 |
+
scopes = emptyList(),
|
| 541 |
+
caps = emptyList(),
|
| 542 |
+
commands = emptyList(),
|
| 543 |
+
permissions = emptyMap(),
|
| 544 |
+
client = buildClientInfo(clientId = "openclaw-control-ui", clientMode = "ui"),
|
| 545 |
+
userAgent = buildUserAgent(),
|
| 546 |
+
)
|
| 547 |
+
}
|
| 548 |
+
|
| 549 |
+
fun refreshGatewayConnection() {
|
| 550 |
+
val endpoint = connectedEndpoint ?: return
|
| 551 |
+
val token = prefs.loadGatewayToken()
|
| 552 |
+
val password = prefs.loadGatewayPassword()
|
| 553 |
+
val tls = resolveTlsParams(endpoint)
|
| 554 |
+
operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls)
|
| 555 |
+
nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls)
|
| 556 |
+
operatorSession.reconnect()
|
| 557 |
+
nodeSession.reconnect()
|
| 558 |
+
}
|
| 559 |
+
|
| 560 |
+
fun connect(endpoint: GatewayEndpoint) {
|
| 561 |
+
connectedEndpoint = endpoint
|
| 562 |
+
operatorStatusText = "Connecting…"
|
| 563 |
+
nodeStatusText = "Connecting…"
|
| 564 |
+
updateStatus()
|
| 565 |
+
val token = prefs.loadGatewayToken()
|
| 566 |
+
val password = prefs.loadGatewayPassword()
|
| 567 |
+
val tls = resolveTlsParams(endpoint)
|
| 568 |
+
operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls)
|
| 569 |
+
nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls)
|
| 570 |
+
}
|
| 571 |
+
|
| 572 |
+
private fun hasRecordAudioPermission(): Boolean {
|
| 573 |
+
return (
|
| 574 |
+
ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) ==
|
| 575 |
+
PackageManager.PERMISSION_GRANTED
|
| 576 |
+
)
|
| 577 |
+
}
|
| 578 |
+
|
| 579 |
+
private fun hasFineLocationPermission(): Boolean {
|
| 580 |
+
return (
|
| 581 |
+
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) ==
|
| 582 |
+
PackageManager.PERMISSION_GRANTED
|
| 583 |
+
)
|
| 584 |
+
}
|
| 585 |
+
|
| 586 |
+
private fun hasCoarseLocationPermission(): Boolean {
|
| 587 |
+
return (
|
| 588 |
+
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) ==
|
| 589 |
+
PackageManager.PERMISSION_GRANTED
|
| 590 |
+
)
|
| 591 |
+
}
|
| 592 |
+
|
| 593 |
+
private fun hasBackgroundLocationPermission(): Boolean {
|
| 594 |
+
return (
|
| 595 |
+
ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
|
| 596 |
+
PackageManager.PERMISSION_GRANTED
|
| 597 |
+
)
|
| 598 |
+
}
|
| 599 |
+
|
| 600 |
+
fun connectManual() {
|
| 601 |
+
val host = manualHost.value.trim()
|
| 602 |
+
val port = manualPort.value
|
| 603 |
+
if (host.isEmpty() || port <= 0 || port > 65535) {
|
| 604 |
+
_statusText.value = "Failed: invalid manual host/port"
|
| 605 |
+
return
|
| 606 |
+
}
|
| 607 |
+
connect(GatewayEndpoint.manual(host = host, port = port))
|
| 608 |
+
}
|
| 609 |
+
|
| 610 |
+
fun disconnect() {
|
| 611 |
+
connectedEndpoint = null
|
| 612 |
+
operatorSession.disconnect()
|
| 613 |
+
nodeSession.disconnect()
|
| 614 |
+
}
|
| 615 |
+
|
| 616 |
+
private fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? {
|
| 617 |
+
val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId)
|
| 618 |
+
val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank()
|
| 619 |
+
val manual = endpoint.stableId.startsWith("manual|")
|
| 620 |
+
|
| 621 |
+
if (manual) {
|
| 622 |
+
if (!manualTls.value) return null
|
| 623 |
+
return GatewayTlsParams(
|
| 624 |
+
required = true,
|
| 625 |
+
expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
|
| 626 |
+
allowTOFU = stored == null,
|
| 627 |
+
stableId = endpoint.stableId,
|
| 628 |
+
)
|
| 629 |
+
}
|
| 630 |
+
|
| 631 |
+
if (hinted) {
|
| 632 |
+
return GatewayTlsParams(
|
| 633 |
+
required = true,
|
| 634 |
+
expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
|
| 635 |
+
allowTOFU = stored == null,
|
| 636 |
+
stableId = endpoint.stableId,
|
| 637 |
+
)
|
| 638 |
+
}
|
| 639 |
+
|
| 640 |
+
if (!stored.isNullOrBlank()) {
|
| 641 |
+
return GatewayTlsParams(
|
| 642 |
+
required = true,
|
| 643 |
+
expectedFingerprint = stored,
|
| 644 |
+
allowTOFU = false,
|
| 645 |
+
stableId = endpoint.stableId,
|
| 646 |
+
)
|
| 647 |
+
}
|
| 648 |
+
|
| 649 |
+
return null
|
| 650 |
+
}
|
| 651 |
+
|
| 652 |
+
fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
|
| 653 |
+
scope.launch {
|
| 654 |
+
val trimmed = payloadJson.trim()
|
| 655 |
+
if (trimmed.isEmpty()) return@launch
|
| 656 |
+
|
| 657 |
+
val root =
|
| 658 |
+
try {
|
| 659 |
+
json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch
|
| 660 |
+
} catch (_: Throwable) {
|
| 661 |
+
return@launch
|
| 662 |
+
}
|
| 663 |
+
|
| 664 |
+
val userActionObj = (root["userAction"] as? JsonObject) ?: root
|
| 665 |
+
val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
|
| 666 |
+
java.util.UUID.randomUUID().toString()
|
| 667 |
+
}
|
| 668 |
+
val name = OpenClawCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch
|
| 669 |
+
|
| 670 |
+
val surfaceId =
|
| 671 |
+
(userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
|
| 672 |
+
val sourceComponentId =
|
| 673 |
+
(userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" }
|
| 674 |
+
val contextJson = (userActionObj["context"] as? JsonObject)?.toString()
|
| 675 |
+
|
| 676 |
+
val sessionKey = resolveMainSessionKey()
|
| 677 |
+
val message =
|
| 678 |
+
OpenClawCanvasA2UIAction.formatAgentMessage(
|
| 679 |
+
actionName = name,
|
| 680 |
+
sessionKey = sessionKey,
|
| 681 |
+
surfaceId = surfaceId,
|
| 682 |
+
sourceComponentId = sourceComponentId,
|
| 683 |
+
host = displayName.value,
|
| 684 |
+
instanceId = instanceId.value.lowercase(),
|
| 685 |
+
contextJson = contextJson,
|
| 686 |
+
)
|
| 687 |
+
|
| 688 |
+
val connected = nodeConnected
|
| 689 |
+
var error: String? = null
|
| 690 |
+
if (connected) {
|
| 691 |
+
try {
|
| 692 |
+
nodeSession.sendNodeEvent(
|
| 693 |
+
event = "agent.request",
|
| 694 |
+
payloadJson =
|
| 695 |
+
buildJsonObject {
|
| 696 |
+
put("message", JsonPrimitive(message))
|
| 697 |
+
put("sessionKey", JsonPrimitive(sessionKey))
|
| 698 |
+
put("thinking", JsonPrimitive("low"))
|
| 699 |
+
put("deliver", JsonPrimitive(false))
|
| 700 |
+
put("key", JsonPrimitive(actionId))
|
| 701 |
+
}.toString(),
|
| 702 |
+
)
|
| 703 |
+
} catch (e: Throwable) {
|
| 704 |
+
error = e.message ?: "send failed"
|
| 705 |
+
}
|
| 706 |
+
} else {
|
| 707 |
+
error = "gateway not connected"
|
| 708 |
+
}
|
| 709 |
+
|
| 710 |
+
try {
|
| 711 |
+
canvas.eval(
|
| 712 |
+
OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus(
|
| 713 |
+
actionId = actionId,
|
| 714 |
+
ok = connected && error == null,
|
| 715 |
+
error = error,
|
| 716 |
+
),
|
| 717 |
+
)
|
| 718 |
+
} catch (_: Throwable) {
|
| 719 |
+
// ignore
|
| 720 |
+
}
|
| 721 |
+
}
|
| 722 |
+
}
|
| 723 |
+
|
| 724 |
+
fun loadChat(sessionKey: String) {
|
| 725 |
+
val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() }
|
| 726 |
+
chat.load(key)
|
| 727 |
+
}
|
| 728 |
+
|
| 729 |
+
fun refreshChat() {
|
| 730 |
+
chat.refresh()
|
| 731 |
+
}
|
| 732 |
+
|
| 733 |
+
fun refreshChatSessions(limit: Int? = null) {
|
| 734 |
+
chat.refreshSessions(limit = limit)
|
| 735 |
+
}
|
| 736 |
+
|
| 737 |
+
fun setChatThinkingLevel(level: String) {
|
| 738 |
+
chat.setThinkingLevel(level)
|
| 739 |
+
}
|
| 740 |
+
|
| 741 |
+
fun switchChatSession(sessionKey: String) {
|
| 742 |
+
chat.switchSession(sessionKey)
|
| 743 |
+
}
|
| 744 |
+
|
| 745 |
+
fun abortChat() {
|
| 746 |
+
chat.abort()
|
| 747 |
+
}
|
| 748 |
+
|
| 749 |
+
fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
|
| 750 |
+
chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments)
|
| 751 |
+
}
|
| 752 |
+
|
| 753 |
+
private fun handleGatewayEvent(event: String, payloadJson: String?) {
|
| 754 |
+
if (event == "voicewake.changed") {
|
| 755 |
+
if (payloadJson.isNullOrBlank()) return
|
| 756 |
+
try {
|
| 757 |
+
val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
|
| 758 |
+
val array = payload["triggers"] as? JsonArray ?: return
|
| 759 |
+
val triggers = array.mapNotNull { it.asStringOrNull() }
|
| 760 |
+
applyWakeWordsFromGateway(triggers)
|
| 761 |
+
} catch (_: Throwable) {
|
| 762 |
+
// ignore
|
| 763 |
+
}
|
| 764 |
+
return
|
| 765 |
+
}
|
| 766 |
+
|
| 767 |
+
talkMode.handleGatewayEvent(event, payloadJson)
|
| 768 |
+
chat.handleGatewayEvent(event, payloadJson)
|
| 769 |
+
}
|
| 770 |
+
|
| 771 |
+
private fun applyWakeWordsFromGateway(words: List<String>) {
|
| 772 |
+
suppressWakeWordsSync = true
|
| 773 |
+
prefs.setWakeWords(words)
|
| 774 |
+
suppressWakeWordsSync = false
|
| 775 |
+
}
|
| 776 |
+
|
| 777 |
+
private fun scheduleWakeWordsSyncIfNeeded() {
|
| 778 |
+
if (suppressWakeWordsSync) return
|
| 779 |
+
if (!_isConnected.value) return
|
| 780 |
+
|
| 781 |
+
val snapshot = prefs.wakeWords.value
|
| 782 |
+
wakeWordsSyncJob?.cancel()
|
| 783 |
+
wakeWordsSyncJob =
|
| 784 |
+
scope.launch {
|
| 785 |
+
delay(650)
|
| 786 |
+
val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() }
|
| 787 |
+
val params = """{"triggers":[$jsonList]}"""
|
| 788 |
+
try {
|
| 789 |
+
operatorSession.request("voicewake.set", params)
|
| 790 |
+
} catch (_: Throwable) {
|
| 791 |
+
// ignore
|
| 792 |
+
}
|
| 793 |
+
}
|
| 794 |
+
}
|
| 795 |
+
|
| 796 |
+
private suspend fun refreshWakeWordsFromGateway() {
|
| 797 |
+
if (!_isConnected.value) return
|
| 798 |
+
try {
|
| 799 |
+
val res = operatorSession.request("voicewake.get", "{}")
|
| 800 |
+
val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return
|
| 801 |
+
val array = payload["triggers"] as? JsonArray ?: return
|
| 802 |
+
val triggers = array.mapNotNull { it.asStringOrNull() }
|
| 803 |
+
applyWakeWordsFromGateway(triggers)
|
| 804 |
+
} catch (_: Throwable) {
|
| 805 |
+
// ignore
|
| 806 |
+
}
|
| 807 |
+
}
|
| 808 |
+
|
| 809 |
+
private suspend fun refreshBrandingFromGateway() {
|
| 810 |
+
if (!_isConnected.value) return
|
| 811 |
+
try {
|
| 812 |
+
val res = operatorSession.request("config.get", "{}")
|
| 813 |
+
val root = json.parseToJsonElement(res).asObjectOrNull()
|
| 814 |
+
val config = root?.get("config").asObjectOrNull()
|
| 815 |
+
val ui = config?.get("ui").asObjectOrNull()
|
| 816 |
+
val raw = ui?.get("seamColor").asStringOrNull()?.trim()
|
| 817 |
+
val sessionCfg = config?.get("session").asObjectOrNull()
|
| 818 |
+
val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull())
|
| 819 |
+
applyMainSessionKey(mainKey)
|
| 820 |
+
|
| 821 |
+
val parsed = parseHexColorArgb(raw)
|
| 822 |
+
_seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB
|
| 823 |
+
} catch (_: Throwable) {
|
| 824 |
+
// ignore
|
| 825 |
+
}
|
| 826 |
+
}
|
| 827 |
+
|
| 828 |
+
private suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
|
| 829 |
+
if (
|
| 830 |
+
command.startsWith(OpenClawCanvasCommand.NamespacePrefix) ||
|
| 831 |
+
command.startsWith(OpenClawCanvasA2UICommand.NamespacePrefix) ||
|
| 832 |
+
command.startsWith(OpenClawCameraCommand.NamespacePrefix) ||
|
| 833 |
+
command.startsWith(OpenClawScreenCommand.NamespacePrefix)
|
| 834 |
+
) {
|
| 835 |
+
if (!isForeground.value) {
|
| 836 |
+
return GatewaySession.InvokeResult.error(
|
| 837 |
+
code = "NODE_BACKGROUND_UNAVAILABLE",
|
| 838 |
+
message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
|
| 839 |
+
)
|
| 840 |
+
}
|
| 841 |
+
}
|
| 842 |
+
if (command.startsWith(OpenClawCameraCommand.NamespacePrefix) && !cameraEnabled.value) {
|
| 843 |
+
return GatewaySession.InvokeResult.error(
|
| 844 |
+
code = "CAMERA_DISABLED",
|
| 845 |
+
message = "CAMERA_DISABLED: enable Camera in Settings",
|
| 846 |
+
)
|
| 847 |
+
}
|
| 848 |
+
if (command.startsWith(OpenClawLocationCommand.NamespacePrefix) &&
|
| 849 |
+
locationMode.value == LocationMode.Off
|
| 850 |
+
) {
|
| 851 |
+
return GatewaySession.InvokeResult.error(
|
| 852 |
+
code = "LOCATION_DISABLED",
|
| 853 |
+
message = "LOCATION_DISABLED: enable Location in Settings",
|
| 854 |
+
)
|
| 855 |
+
}
|
| 856 |
+
|
| 857 |
+
return when (command) {
|
| 858 |
+
OpenClawCanvasCommand.Present.rawValue -> {
|
| 859 |
+
val url = CanvasController.parseNavigateUrl(paramsJson)
|
| 860 |
+
canvas.navigate(url)
|
| 861 |
+
GatewaySession.InvokeResult.ok(null)
|
| 862 |
+
}
|
| 863 |
+
OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null)
|
| 864 |
+
OpenClawCanvasCommand.Navigate.rawValue -> {
|
| 865 |
+
val url = CanvasController.parseNavigateUrl(paramsJson)
|
| 866 |
+
canvas.navigate(url)
|
| 867 |
+
GatewaySession.InvokeResult.ok(null)
|
| 868 |
+
}
|
| 869 |
+
OpenClawCanvasCommand.Eval.rawValue -> {
|
| 870 |
+
val js =
|
| 871 |
+
CanvasController.parseEvalJs(paramsJson)
|
| 872 |
+
?: return GatewaySession.InvokeResult.error(
|
| 873 |
+
code = "INVALID_REQUEST",
|
| 874 |
+
message = "INVALID_REQUEST: javaScript required",
|
| 875 |
+
)
|
| 876 |
+
val result =
|
| 877 |
+
try {
|
| 878 |
+
canvas.eval(js)
|
| 879 |
+
} catch (err: Throwable) {
|
| 880 |
+
return GatewaySession.InvokeResult.error(
|
| 881 |
+
code = "NODE_BACKGROUND_UNAVAILABLE",
|
| 882 |
+
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
|
| 883 |
+
)
|
| 884 |
+
}
|
| 885 |
+
GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
|
| 886 |
+
}
|
| 887 |
+
OpenClawCanvasCommand.Snapshot.rawValue -> {
|
| 888 |
+
val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
|
| 889 |
+
val base64 =
|
| 890 |
+
try {
|
| 891 |
+
canvas.snapshotBase64(
|
| 892 |
+
format = snapshotParams.format,
|
| 893 |
+
quality = snapshotParams.quality,
|
| 894 |
+
maxWidth = snapshotParams.maxWidth,
|
| 895 |
+
)
|
| 896 |
+
} catch (err: Throwable) {
|
| 897 |
+
return GatewaySession.InvokeResult.error(
|
| 898 |
+
code = "NODE_BACKGROUND_UNAVAILABLE",
|
| 899 |
+
message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
|
| 900 |
+
)
|
| 901 |
+
}
|
| 902 |
+
GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
|
| 903 |
+
}
|
| 904 |
+
OpenClawCanvasA2UICommand.Reset.rawValue -> {
|
| 905 |
+
val a2uiUrl = resolveA2uiHostUrl()
|
| 906 |
+
?: return GatewaySession.InvokeResult.error(
|
| 907 |
+
code = "A2UI_HOST_NOT_CONFIGURED",
|
| 908 |
+
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
|
| 909 |
+
)
|
| 910 |
+
val ready = ensureA2uiReady(a2uiUrl)
|
| 911 |
+
if (!ready) {
|
| 912 |
+
return GatewaySession.InvokeResult.error(
|
| 913 |
+
code = "A2UI_HOST_UNAVAILABLE",
|
| 914 |
+
message = "A2UI host not reachable",
|
| 915 |
+
)
|
| 916 |
+
}
|
| 917 |
+
val res = canvas.eval(a2uiResetJS)
|
| 918 |
+
GatewaySession.InvokeResult.ok(res)
|
| 919 |
+
}
|
| 920 |
+
OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
|
| 921 |
+
val messages =
|
| 922 |
+
try {
|
| 923 |
+
decodeA2uiMessages(command, paramsJson)
|
| 924 |
+
} catch (err: Throwable) {
|
| 925 |
+
return GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload")
|
| 926 |
+
}
|
| 927 |
+
val a2uiUrl = resolveA2uiHostUrl()
|
| 928 |
+
?: return GatewaySession.InvokeResult.error(
|
| 929 |
+
code = "A2UI_HOST_NOT_CONFIGURED",
|
| 930 |
+
message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
|
| 931 |
+
)
|
| 932 |
+
val ready = ensureA2uiReady(a2uiUrl)
|
| 933 |
+
if (!ready) {
|
| 934 |
+
return GatewaySession.InvokeResult.error(
|
| 935 |
+
code = "A2UI_HOST_UNAVAILABLE",
|
| 936 |
+
message = "A2UI host not reachable",
|
| 937 |
+
)
|
| 938 |
+
}
|
| 939 |
+
val js = a2uiApplyMessagesJS(messages)
|
| 940 |
+
val res = canvas.eval(js)
|
| 941 |
+
GatewaySession.InvokeResult.ok(res)
|
| 942 |
+
}
|
| 943 |
+
OpenClawCameraCommand.Snap.rawValue -> {
|
| 944 |
+
showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo)
|
| 945 |
+
triggerCameraFlash()
|
| 946 |
+
val res =
|
| 947 |
+
try {
|
| 948 |
+
camera.snap(paramsJson)
|
| 949 |
+
} catch (err: Throwable) {
|
| 950 |
+
val (code, message) = invokeErrorFromThrowable(err)
|
| 951 |
+
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200)
|
| 952 |
+
return GatewaySession.InvokeResult.error(code = code, message = message)
|
| 953 |
+
}
|
| 954 |
+
showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600)
|
| 955 |
+
GatewaySession.InvokeResult.ok(res.payloadJson)
|
| 956 |
+
}
|
| 957 |
+
OpenClawCameraCommand.Clip.rawValue -> {
|
| 958 |
+
val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false
|
| 959 |
+
if (includeAudio) externalAudioCaptureActive.value = true
|
| 960 |
+
try {
|
| 961 |
+
showCameraHud(message = "Recording…", kind = CameraHudKind.Recording)
|
| 962 |
+
val res =
|
| 963 |
+
try {
|
| 964 |
+
camera.clip(paramsJson)
|
| 965 |
+
} catch (err: Throwable) {
|
| 966 |
+
val (code, message) = invokeErrorFromThrowable(err)
|
| 967 |
+
showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400)
|
| 968 |
+
return GatewaySession.InvokeResult.error(code = code, message = message)
|
| 969 |
+
}
|
| 970 |
+
showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800)
|
| 971 |
+
GatewaySession.InvokeResult.ok(res.payloadJson)
|
| 972 |
+
} finally {
|
| 973 |
+
if (includeAudio) externalAudioCaptureActive.value = false
|
| 974 |
+
}
|
| 975 |
+
}
|
| 976 |
+
OpenClawLocationCommand.Get.rawValue -> {
|
| 977 |
+
val mode = locationMode.value
|
| 978 |
+
if (!isForeground.value && mode != LocationMode.Always) {
|
| 979 |
+
return GatewaySession.InvokeResult.error(
|
| 980 |
+
code = "LOCATION_BACKGROUND_UNAVAILABLE",
|
| 981 |
+
message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always",
|
| 982 |
+
)
|
| 983 |
+
}
|
| 984 |
+
if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) {
|
| 985 |
+
return GatewaySession.InvokeResult.error(
|
| 986 |
+
code = "LOCATION_PERMISSION_REQUIRED",
|
| 987 |
+
message = "LOCATION_PERMISSION_REQUIRED: grant Location permission",
|
| 988 |
+
)
|
| 989 |
+
}
|
| 990 |
+
if (!isForeground.value && mode == LocationMode.Always && !hasBackgroundLocationPermission()) {
|
| 991 |
+
return GatewaySession.InvokeResult.error(
|
| 992 |
+
code = "LOCATION_PERMISSION_REQUIRED",
|
| 993 |
+
message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings",
|
| 994 |
+
)
|
| 995 |
+
}
|
| 996 |
+
val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson)
|
| 997 |
+
val preciseEnabled = locationPreciseEnabled.value
|
| 998 |
+
val accuracy =
|
| 999 |
+
when (desiredAccuracy) {
|
| 1000 |
+
"precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
|
| 1001 |
+
"coarse" -> "coarse"
|
| 1002 |
+
else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
|
| 1003 |
+
}
|
| 1004 |
+
val providers =
|
| 1005 |
+
when (accuracy) {
|
| 1006 |
+
"precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER)
|
| 1007 |
+
"coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
|
| 1008 |
+
else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
|
| 1009 |
+
}
|
| 1010 |
+
try {
|
| 1011 |
+
val payload =
|
| 1012 |
+
location.getLocation(
|
| 1013 |
+
desiredProviders = providers,
|
| 1014 |
+
maxAgeMs = maxAgeMs,
|
| 1015 |
+
timeoutMs = timeoutMs,
|
| 1016 |
+
isPrecise = accuracy == "precise",
|
| 1017 |
+
)
|
| 1018 |
+
GatewaySession.InvokeResult.ok(payload.payloadJson)
|
| 1019 |
+
} catch (err: TimeoutCancellationException) {
|
| 1020 |
+
GatewaySession.InvokeResult.error(
|
| 1021 |
+
code = "LOCATION_TIMEOUT",
|
| 1022 |
+
message = "LOCATION_TIMEOUT: no fix in time",
|
| 1023 |
+
)
|
| 1024 |
+
} catch (err: Throwable) {
|
| 1025 |
+
val message = err.message ?: "LOCATION_UNAVAILABLE: no fix"
|
| 1026 |
+
GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message)
|
| 1027 |
+
}
|
| 1028 |
+
}
|
| 1029 |
+
OpenClawScreenCommand.Record.rawValue -> {
|
| 1030 |
+
// Status pill mirrors screen recording state so it stays visible without overlay stacking.
|
| 1031 |
+
_screenRecordActive.value = true
|
| 1032 |
+
try {
|
| 1033 |
+
val res =
|
| 1034 |
+
try {
|
| 1035 |
+
screenRecorder.record(paramsJson)
|
| 1036 |
+
} catch (err: Throwable) {
|
| 1037 |
+
val (code, message) = invokeErrorFromThrowable(err)
|
| 1038 |
+
return GatewaySession.InvokeResult.error(code = code, message = message)
|
| 1039 |
+
}
|
| 1040 |
+
GatewaySession.InvokeResult.ok(res.payloadJson)
|
| 1041 |
+
} finally {
|
| 1042 |
+
_screenRecordActive.value = false
|
| 1043 |
+
}
|
| 1044 |
+
}
|
| 1045 |
+
OpenClawSmsCommand.Send.rawValue -> {
|
| 1046 |
+
val res = sms.send(paramsJson)
|
| 1047 |
+
if (res.ok) {
|
| 1048 |
+
GatewaySession.InvokeResult.ok(res.payloadJson)
|
| 1049 |
+
} else {
|
| 1050 |
+
val error = res.error ?: "SMS_SEND_FAILED"
|
| 1051 |
+
val idx = error.indexOf(':')
|
| 1052 |
+
val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED"
|
| 1053 |
+
GatewaySession.InvokeResult.error(code = code, message = error)
|
| 1054 |
+
}
|
| 1055 |
+
}
|
| 1056 |
+
else ->
|
| 1057 |
+
GatewaySession.InvokeResult.error(
|
| 1058 |
+
code = "INVALID_REQUEST",
|
| 1059 |
+
message = "INVALID_REQUEST: unknown command",
|
| 1060 |
+
)
|
| 1061 |
+
}
|
| 1062 |
+
}
|
| 1063 |
+
|
| 1064 |
+
private fun triggerCameraFlash() {
|
| 1065 |
+
// Token is used as a pulse trigger; value doesn't matter as long as it changes.
|
| 1066 |
+
_cameraFlashToken.value = SystemClock.elapsedRealtimeNanos()
|
| 1067 |
+
}
|
| 1068 |
+
|
| 1069 |
+
private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) {
|
| 1070 |
+
val token = cameraHudSeq.incrementAndGet()
|
| 1071 |
+
_cameraHud.value = CameraHudState(token = token, kind = kind, message = message)
|
| 1072 |
+
|
| 1073 |
+
if (autoHideMs != null && autoHideMs > 0) {
|
| 1074 |
+
scope.launch {
|
| 1075 |
+
delay(autoHideMs)
|
| 1076 |
+
if (_cameraHud.value?.token == token) _cameraHud.value = null
|
| 1077 |
+
}
|
| 1078 |
+
}
|
| 1079 |
+
}
|
| 1080 |
+
|
| 1081 |
+
private fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
|
| 1082 |
+
val raw = (err.message ?: "").trim()
|
| 1083 |
+
if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error"
|
| 1084 |
+
|
| 1085 |
+
val idx = raw.indexOf(':')
|
| 1086 |
+
if (idx <= 0) return "UNAVAILABLE" to raw
|
| 1087 |
+
val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" }
|
| 1088 |
+
val message = raw.substring(idx + 1).trim().ifEmpty { raw }
|
| 1089 |
+
// Preserve full string for callers/logging, but keep the returned message human-friendly.
|
| 1090 |
+
return code to "$code: $message"
|
| 1091 |
+
}
|
| 1092 |
+
|
| 1093 |
+
private fun parseLocationParams(paramsJson: String?): Triple<Long?, Long, String?> {
|
| 1094 |
+
if (paramsJson.isNullOrBlank()) {
|
| 1095 |
+
return Triple(null, 10_000L, null)
|
| 1096 |
+
}
|
| 1097 |
+
val root =
|
| 1098 |
+
try {
|
| 1099 |
+
json.parseToJsonElement(paramsJson).asObjectOrNull()
|
| 1100 |
+
} catch (_: Throwable) {
|
| 1101 |
+
null
|
| 1102 |
+
}
|
| 1103 |
+
val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull()
|
| 1104 |
+
val timeoutMs =
|
| 1105 |
+
(root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L)
|
| 1106 |
+
?: 10_000L
|
| 1107 |
+
val desiredAccuracy =
|
| 1108 |
+
(root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase()
|
| 1109 |
+
return Triple(maxAgeMs, timeoutMs, desiredAccuracy)
|
| 1110 |
+
}
|
| 1111 |
+
|
| 1112 |
+
private fun resolveA2uiHostUrl(): String? {
|
| 1113 |
+
val nodeRaw = nodeSession.currentCanvasHostUrl()?.trim().orEmpty()
|
| 1114 |
+
val operatorRaw = operatorSession.currentCanvasHostUrl()?.trim().orEmpty()
|
| 1115 |
+
val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw
|
| 1116 |
+
if (raw.isBlank()) return null
|
| 1117 |
+
val base = raw.trimEnd('/')
|
| 1118 |
+
return "${base}/__openclaw__/a2ui/?platform=android"
|
| 1119 |
+
}
|
| 1120 |
+
|
| 1121 |
+
private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
|
| 1122 |
+
try {
|
| 1123 |
+
val already = canvas.eval(a2uiReadyCheckJS)
|
| 1124 |
+
if (already == "true") return true
|
| 1125 |
+
} catch (_: Throwable) {
|
| 1126 |
+
// ignore
|
| 1127 |
+
}
|
| 1128 |
+
|
| 1129 |
+
canvas.navigate(a2uiUrl)
|
| 1130 |
+
repeat(50) {
|
| 1131 |
+
try {
|
| 1132 |
+
val ready = canvas.eval(a2uiReadyCheckJS)
|
| 1133 |
+
if (ready == "true") return true
|
| 1134 |
+
} catch (_: Throwable) {
|
| 1135 |
+
// ignore
|
| 1136 |
+
}
|
| 1137 |
+
delay(120)
|
| 1138 |
+
}
|
| 1139 |
+
return false
|
| 1140 |
+
}
|
| 1141 |
+
|
| 1142 |
+
private fun decodeA2uiMessages(command: String, paramsJson: String?): String {
|
| 1143 |
+
val raw = paramsJson?.trim().orEmpty()
|
| 1144 |
+
if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
|
| 1145 |
+
|
| 1146 |
+
val obj =
|
| 1147 |
+
json.parseToJsonElement(raw) as? JsonObject
|
| 1148 |
+
?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
|
| 1149 |
+
|
| 1150 |
+
val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
|
| 1151 |
+
val hasMessagesArray = obj["messages"] is JsonArray
|
| 1152 |
+
|
| 1153 |
+
if (command == OpenClawCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) {
|
| 1154 |
+
val jsonl = jsonlField
|
| 1155 |
+
if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
|
| 1156 |
+
val messages =
|
| 1157 |
+
jsonl
|
| 1158 |
+
.lineSequence()
|
| 1159 |
+
.map { it.trim() }
|
| 1160 |
+
.filter { it.isNotBlank() }
|
| 1161 |
+
.mapIndexed { idx, line ->
|
| 1162 |
+
val el = json.parseToJsonElement(line)
|
| 1163 |
+
val msg =
|
| 1164 |
+
el as? JsonObject
|
| 1165 |
+
?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
|
| 1166 |
+
validateA2uiV0_8(msg, idx + 1)
|
| 1167 |
+
msg
|
| 1168 |
+
}
|
| 1169 |
+
.toList()
|
| 1170 |
+
return JsonArray(messages).toString()
|
| 1171 |
+
}
|
| 1172 |
+
|
| 1173 |
+
val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
|
| 1174 |
+
val out =
|
| 1175 |
+
arr.mapIndexed { idx, el ->
|
| 1176 |
+
val msg =
|
| 1177 |
+
el as? JsonObject
|
| 1178 |
+
?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
|
| 1179 |
+
validateA2uiV0_8(msg, idx + 1)
|
| 1180 |
+
msg
|
| 1181 |
+
}
|
| 1182 |
+
return JsonArray(out).toString()
|
| 1183 |
+
}
|
| 1184 |
+
|
| 1185 |
+
private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
|
| 1186 |
+
if (msg.containsKey("createSurface")) {
|
| 1187 |
+
throw IllegalArgumentException(
|
| 1188 |
+
"A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
|
| 1189 |
+
)
|
| 1190 |
+
}
|
| 1191 |
+
val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
|
| 1192 |
+
val matched = msg.keys.filter { allowed.contains(it) }
|
| 1193 |
+
if (matched.size != 1) {
|
| 1194 |
+
val found = msg.keys.sorted().joinToString(", ")
|
| 1195 |
+
throw IllegalArgumentException(
|
| 1196 |
+
"A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
|
| 1197 |
+
)
|
| 1198 |
+
}
|
| 1199 |
+
}
|
| 1200 |
+
}
|
| 1201 |
+
|
| 1202 |
+
private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
|
| 1203 |
+
|
| 1204 |
+
private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
|
| 1205 |
+
|
| 1206 |
+
private const val a2uiReadyCheckJS: String =
|
| 1207 |
+
"""
|
| 1208 |
+
(() => {
|
| 1209 |
+
try {
|
| 1210 |
+
const host = globalThis.openclawA2UI;
|
| 1211 |
+
return !!host && typeof host.applyMessages === 'function';
|
| 1212 |
+
} catch (_) {
|
| 1213 |
+
return false;
|
| 1214 |
+
}
|
| 1215 |
+
})()
|
| 1216 |
+
"""
|
| 1217 |
+
|
| 1218 |
+
private const val a2uiResetJS: String =
|
| 1219 |
+
"""
|
| 1220 |
+
(() => {
|
| 1221 |
+
try {
|
| 1222 |
+
const host = globalThis.openclawA2UI;
|
| 1223 |
+
if (!host) return { ok: false, error: "missing openclawA2UI" };
|
| 1224 |
+
return host.reset();
|
| 1225 |
+
} catch (e) {
|
| 1226 |
+
return { ok: false, error: String(e?.message ?? e) };
|
| 1227 |
+
}
|
| 1228 |
+
})()
|
| 1229 |
+
"""
|
| 1230 |
+
|
| 1231 |
+
private fun a2uiApplyMessagesJS(messagesJson: String): String {
|
| 1232 |
+
return """
|
| 1233 |
+
(() => {
|
| 1234 |
+
try {
|
| 1235 |
+
const host = globalThis.openclawA2UI;
|
| 1236 |
+
if (!host) return { ok: false, error: "missing openclawA2UI" };
|
| 1237 |
+
const messages = $messagesJson;
|
| 1238 |
+
return host.applyMessages(messages);
|
| 1239 |
+
} catch (e) {
|
| 1240 |
+
return { ok: false, error: String(e?.message ?? e) };
|
| 1241 |
+
}
|
| 1242 |
+
})()
|
| 1243 |
+
""".trimIndent()
|
| 1244 |
+
}
|
| 1245 |
+
|
| 1246 |
+
private fun String.toJsonString(): String {
|
| 1247 |
+
val escaped =
|
| 1248 |
+
this.replace("\\", "\\\\")
|
| 1249 |
+
.replace("\"", "\\\"")
|
| 1250 |
+
.replace("\n", "\\n")
|
| 1251 |
+
.replace("\r", "\\r")
|
| 1252 |
+
return "\"$escaped\""
|
| 1253 |
+
}
|
| 1254 |
+
|
| 1255 |
+
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
|
| 1256 |
+
|
| 1257 |
+
private fun JsonElement?.asStringOrNull(): String? =
|
| 1258 |
+
when (this) {
|
| 1259 |
+
is JsonNull -> null
|
| 1260 |
+
is JsonPrimitive -> content
|
| 1261 |
+
else -> null
|
| 1262 |
+
}
|
| 1263 |
+
|
| 1264 |
+
private fun parseHexColorArgb(raw: String?): Long? {
|
| 1265 |
+
val trimmed = raw?.trim().orEmpty()
|
| 1266 |
+
if (trimmed.isEmpty()) return null
|
| 1267 |
+
val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
|
| 1268 |
+
if (hex.length != 6) return null
|
| 1269 |
+
val rgb = hex.toLongOrNull(16) ?: return null
|
| 1270 |
+
return 0xFF000000L or rgb
|
| 1271 |
+
}
|
apps/android/app/src/main/java/ai/openclaw/android/PermissionRequester.kt
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package ai.openclaw.android
|
| 2 |
+
|
| 3 |
+
import android.content.pm.PackageManager
|
| 4 |
+
import android.content.Intent
|
| 5 |
+
import android.Manifest
|
| 6 |
+
import android.net.Uri
|
| 7 |
+
import android.provider.Settings
|
| 8 |
+
import androidx.appcompat.app.AlertDialog
|
| 9 |
+
import androidx.activity.ComponentActivity
|
| 10 |
+
import androidx.activity.result.ActivityResultLauncher
|
| 11 |
+
import androidx.activity.result.contract.ActivityResultContracts
|
| 12 |
+
import androidx.core.content.ContextCompat
|
| 13 |
+
import androidx.core.app.ActivityCompat
|
| 14 |
+
import kotlinx.coroutines.CompletableDeferred
|
| 15 |
+
import kotlinx.coroutines.Dispatchers
|
| 16 |
+
import kotlinx.coroutines.sync.Mutex
|
| 17 |
+
import kotlinx.coroutines.sync.withLock
|
| 18 |
+
import kotlinx.coroutines.withContext
|
| 19 |
+
import kotlinx.coroutines.suspendCancellableCoroutine
|
| 20 |
+
import kotlin.coroutines.resume
|
| 21 |
+
|
| 22 |
+
class PermissionRequester(private val activity: ComponentActivity) {
|
| 23 |
+
private val mutex = Mutex()
|
| 24 |
+
private var pending: CompletableDeferred<Map<String, Boolean>>? = null
|
| 25 |
+
|
| 26 |
+
private val launcher: ActivityResultLauncher<Array<String>> =
|
| 27 |
+
activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result ->
|
| 28 |
+
val p = pending
|
| 29 |
+
pending = null
|
| 30 |
+
p?.complete(result)
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
suspend fun requestIfMissing(
|
| 34 |
+
permissions: List<String>,
|
| 35 |
+
timeoutMs: Long = 20_000,
|
| 36 |
+
): Map<String, Boolean> =
|
| 37 |
+
mutex.withLock {
|
| 38 |
+
val missing =
|
| 39 |
+
permissions.filter { perm ->
|
| 40 |
+
ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED
|
| 41 |
+
}
|
| 42 |
+
if (missing.isEmpty()) {
|
| 43 |
+
return permissions.associateWith { true }
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
val needsRationale =
|
| 47 |
+
missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) }
|
| 48 |
+
if (needsRationale) {
|
| 49 |
+
val proceed = showRationaleDialog(missing)
|
| 50 |
+
if (!proceed) {
|
| 51 |
+
return permissions.associateWith { perm ->
|
| 52 |
+
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
val deferred = CompletableDeferred<Map<String, Boolean>>()
|
| 58 |
+
pending = deferred
|
| 59 |
+
withContext(Dispatchers.Main) {
|
| 60 |
+
launcher.launch(missing.toTypedArray())
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
val result =
|
| 64 |
+
withContext(Dispatchers.Default) {
|
| 65 |
+
kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() }
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
// Merge: if something was already granted, treat it as granted even if launcher omitted it.
|
| 69 |
+
val merged =
|
| 70 |
+
permissions.associateWith { perm ->
|
| 71 |
+
val nowGranted =
|
| 72 |
+
ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
|
| 73 |
+
result[perm] == true || nowGranted
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
val denied =
|
| 77 |
+
merged.filterValues { !it }.keys.filter {
|
| 78 |
+
!ActivityCompat.shouldShowRequestPermissionRationale(activity, it)
|
| 79 |
+
}
|
| 80 |
+
if (denied.isNotEmpty()) {
|
| 81 |
+
showSettingsDialog(denied)
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
return merged
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
private suspend fun showRationaleDialog(permissions: List<String>): Boolean =
|
| 88 |
+
withContext(Dispatchers.Main) {
|
| 89 |
+
suspendCancellableCoroutine { cont ->
|
| 90 |
+
AlertDialog.Builder(activity)
|
| 91 |
+
.setTitle("Permission required")
|
| 92 |
+
.setMessage(buildRationaleMessage(permissions))
|
| 93 |
+
.setPositiveButton("Continue") { _, _ -> cont.resume(true) }
|
| 94 |
+
.setNegativeButton("Not now") { _, _ -> cont.resume(false) }
|
| 95 |
+
.setOnCancelListener { cont.resume(false) }
|
| 96 |
+
.show()
|
| 97 |
+
}
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
private fun showSettingsDialog(permissions: List<String>) {
|
| 101 |
+
AlertDialog.Builder(activity)
|
| 102 |
+
.setTitle("Enable permission in Settings")
|
| 103 |
+
.setMessage(buildSettingsMessage(permissions))
|
| 104 |
+
.setPositiveButton("Open Settings") { _, _ ->
|
| 105 |
+
val intent =
|
| 106 |
+
Intent(
|
| 107 |
+
Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
|
| 108 |
+
Uri.fromParts("package", activity.packageName, null),
|
| 109 |
+
)
|
| 110 |
+
activity.startActivity(intent)
|
| 111 |
+
}
|
| 112 |
+
.setNegativeButton("Cancel", null)
|
| 113 |
+
.show()
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
private fun buildRationaleMessage(permissions: List<String>): String {
|
| 117 |
+
val labels = permissions.map { permissionLabel(it) }
|
| 118 |
+
return "OpenClaw needs ${labels.joinToString(", ")} permissions to continue."
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
private fun buildSettingsMessage(permissions: List<String>): String {
|
| 122 |
+
val labels = permissions.map { permissionLabel(it) }
|
| 123 |
+
return "Please enable ${labels.joinToString(", ")} in Android Settings to continue."
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
private fun permissionLabel(permission: String): String =
|
| 127 |
+
when (permission) {
|
| 128 |
+
Manifest.permission.CAMERA -> "Camera"
|
| 129 |
+
Manifest.permission.RECORD_AUDIO -> "Microphone"
|
| 130 |
+
Manifest.permission.SEND_SMS -> "SMS"
|
| 131 |
+
else -> permission
|
| 132 |
+
}
|
| 133 |
+
}
|