darkfire514 commited on
Commit
4fc4790
·
verified ·
1 Parent(s): 7206285

Upload 581 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +5 -0
  2. Swabble/.github/workflows/ci.yml +54 -0
  3. Swabble/.gitignore +33 -0
  4. Swabble/.swiftformat +8 -0
  5. Swabble/.swiftlint.yml +43 -0
  6. Swabble/CHANGELOG.md +11 -0
  7. Swabble/LICENSE +21 -0
  8. Swabble/Package.resolved +69 -0
  9. Swabble/Package.swift +55 -0
  10. Swabble/README.md +111 -0
  11. Swabble/Sources/SwabbleCore/Config/Config.swift +77 -0
  12. Swabble/Sources/SwabbleCore/Hooks/HookExecutor.swift +75 -0
  13. Swabble/Sources/SwabbleCore/Speech/BufferConverter.swift +50 -0
  14. Swabble/Sources/SwabbleCore/Speech/SpeechPipeline.swift +114 -0
  15. Swabble/Sources/SwabbleCore/Support/AttributedString+Sentences.swift +62 -0
  16. Swabble/Sources/SwabbleCore/Support/Logging.swift +41 -0
  17. Swabble/Sources/SwabbleCore/Support/OutputFormat.swift +45 -0
  18. Swabble/Sources/SwabbleCore/Support/TranscriptsStore.swift +45 -0
  19. Swabble/Sources/SwabbleKit/WakeWordGate.swift +197 -0
  20. Swabble/Sources/swabble/CLI/CLIRegistry.swift +71 -0
  21. Swabble/Sources/swabble/Commands/DoctorCommand.swift +37 -0
  22. Swabble/Sources/swabble/Commands/HealthCommand.swift +16 -0
  23. Swabble/Sources/swabble/Commands/MicCommands.swift +62 -0
  24. Swabble/Sources/swabble/Commands/ServeCommand.swift +81 -0
  25. Swabble/Sources/swabble/Commands/ServiceCommands.swift +77 -0
  26. Swabble/Sources/swabble/Commands/SetupCommand.swift +26 -0
  27. Swabble/Sources/swabble/Commands/StartStopCommands.swift +35 -0
  28. Swabble/Sources/swabble/Commands/StatusCommand.swift +34 -0
  29. Swabble/Sources/swabble/Commands/TailLogCommand.swift +20 -0
  30. Swabble/Sources/swabble/Commands/TestHookCommand.swift +30 -0
  31. Swabble/Sources/swabble/Commands/TranscribeCommand.swift +61 -0
  32. Swabble/Sources/swabble/main.swift +151 -0
  33. Swabble/Tests/SwabbleKitTests/WakeWordGateTests.swift +63 -0
  34. Swabble/Tests/swabbleTests/ConfigTests.swift +23 -0
  35. Swabble/docs/spec.md +33 -0
  36. Swabble/scripts/format.sh +5 -0
  37. Swabble/scripts/lint.sh +9 -0
  38. apps/android/.gitignore +5 -0
  39. apps/android/README.md +51 -0
  40. apps/android/app/build.gradle.kts +128 -0
  41. apps/android/app/src/main/AndroidManifest.xml +49 -0
  42. apps/android/app/src/main/java/ai/openclaw/android/CameraHudState.kt +14 -0
  43. apps/android/app/src/main/java/ai/openclaw/android/DeviceNames.kt +26 -0
  44. apps/android/app/src/main/java/ai/openclaw/android/LocationMode.kt +15 -0
  45. apps/android/app/src/main/java/ai/openclaw/android/MainActivity.kt +130 -0
  46. apps/android/app/src/main/java/ai/openclaw/android/MainViewModel.kt +174 -0
  47. apps/android/app/src/main/java/ai/openclaw/android/NodeApp.kt +26 -0
  48. apps/android/app/src/main/java/ai/openclaw/android/NodeForegroundService.kt +180 -0
  49. apps/android/app/src/main/java/ai/openclaw/android/NodeRuntime.kt +1271 -0
  50. apps/android/app/src/main/java/ai/openclaw/android/PermissionRequester.kt +133 -0
.gitattributes CHANGED
@@ -1,2 +1,7 @@
1
  * text=auto eol=lf
2
  README-header.png filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
1
  * text=auto eol=lf
2
  README-header.png filter=lfs diff=lfs merge=lfs -text
3
+ apps/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png filter=lfs diff=lfs merge=lfs -text
4
+ apps/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png filter=lfs diff=lfs merge=lfs -text
5
+ apps/ios/Sources/Assets.xcassets/AppIcon.appiconset/icon-1024.png filter=lfs diff=lfs merge=lfs -text
6
+ apps/macos/Icon.icon/Assets/openclaw-mac.png filter=lfs diff=lfs merge=lfs -text
7
+ apps/macos/Sources/OpenClaw/Resources/OpenClaw.icns filter=lfs diff=lfs merge=lfs -text
Swabble/.github/workflows/ci.yml ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ build-and-test:
10
+ runs-on: macos-latest
11
+ defaults:
12
+ run:
13
+ shell: bash
14
+ working-directory: swabble
15
+ steps:
16
+ - name: Checkout swabble
17
+ uses: actions/checkout@v4
18
+ with:
19
+ path: swabble
20
+
21
+ - name: Select Xcode 26.1 (prefer 26.1.1)
22
+ run: |
23
+ set -euo pipefail
24
+ # pick the newest installed 26.1.x, fallback to newest 26.x
25
+ CANDIDATE="$(ls -d /Applications/Xcode_26.1*.app 2>/dev/null | sort -V | tail -1 || true)"
26
+ if [[ -z "$CANDIDATE" ]]; then
27
+ CANDIDATE="$(ls -d /Applications/Xcode_26*.app 2>/dev/null | sort -V | tail -1 || true)"
28
+ fi
29
+ if [[ -z "$CANDIDATE" ]]; then
30
+ echo "No Xcode 26.x found on runner" >&2
31
+ exit 1
32
+ fi
33
+ echo "Selecting $CANDIDATE"
34
+ sudo xcode-select -s "$CANDIDATE"
35
+ xcodebuild -version
36
+
37
+ - name: Show Swift version
38
+ run: swift --version
39
+
40
+ - name: Install tooling
41
+ run: |
42
+ brew update
43
+ brew install swiftlint swiftformat
44
+
45
+ - name: Format check
46
+ run: |
47
+ ./scripts/format.sh
48
+ git diff --exit-code
49
+
50
+ - name: Lint
51
+ run: ./scripts/lint.sh
52
+
53
+ - name: Test
54
+ run: swift test --parallel
Swabble/.gitignore ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # macOS
2
+ .DS_Store
3
+
4
+ # SwiftPM / Build
5
+ /.build
6
+ /.swiftpm
7
+ /DerivedData
8
+ xcuserdata/
9
+ *.xcuserstate
10
+
11
+ # Editors
12
+ /.vscode
13
+ .idea/
14
+
15
+ # Xcode artifacts
16
+ *.hmap
17
+ *.ipa
18
+ *.dSYM.zip
19
+ *.dSYM
20
+
21
+ # Playgrounds
22
+ *.xcplayground
23
+ playground.xcworkspace
24
+ timeline.xctimeline
25
+
26
+ # Carthage
27
+ Carthage/Build/
28
+
29
+ # fastlane
30
+ fastlane/report.xml
31
+ fastlane/Preview.html
32
+ fastlane/screenshots/**/*.png
33
+ fastlane/test_output
Swabble/.swiftformat ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ --swiftversion 6.2
2
+ --indent 4
3
+ --maxwidth 120
4
+ --wraparguments before-first
5
+ --wrapcollections before-first
6
+ --stripunusedargs closure-only
7
+ --self remove
8
+ --header ""
Swabble/.swiftlint.yml ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SwiftLint for swabble
2
+ included:
3
+ - Sources
4
+ excluded:
5
+ - .build
6
+ - DerivedData
7
+ - "**/.swiftpm"
8
+ - "**/.build"
9
+ - "**/DerivedData"
10
+ - "**/.DS_Store"
11
+ opt_in_rules:
12
+ - array_init
13
+ - closure_spacing
14
+ - explicit_init
15
+ - fatal_error_message
16
+ - first_where
17
+ - joined_default_parameter
18
+ - last_where
19
+ - literal_expression_end_indentation
20
+ - multiline_arguments
21
+ - multiline_parameters
22
+ - operator_usage_whitespace
23
+ - redundant_nil_coalescing
24
+ - sorted_first_last
25
+ - switch_case_alignment
26
+ - vertical_parameter_alignment_on_call
27
+ - vertical_whitespace_opening_braces
28
+ - vertical_whitespace_closing_braces
29
+
30
+ disabled_rules:
31
+ - trailing_whitespace
32
+ - trailing_newline
33
+ - indentation_width
34
+ - identifier_name
35
+ - explicit_self
36
+ - file_header
37
+ - todo
38
+
39
+ line_length:
40
+ warning: 140
41
+ error: 180
42
+
43
+ reporter: "xcode"
Swabble/CHANGELOG.md ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Changelog
2
+
3
+ ## 0.2.0 — 2025-12-23
4
+
5
+ ### Highlights
6
+ - Added `SwabbleKit` (multi-platform wake-word gate utilities with segment-aware gap detection).
7
+ - Swabble package now supports iOS + macOS consumers; CLI remains macOS 26-only.
8
+
9
+ ### Changes
10
+ - CLI wake-word matching/stripping routed through `SwabbleKit` helpers.
11
+ - Speech pipeline types now explicitly gated to macOS 26 / iOS 26 availability.
Swabble/LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Peter Steinberger
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
Swabble/Package.resolved ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "originHash" : "24a723309d7a0039d3df3051106f77ac1ed7068a02508e3a6804e41d757e6c72",
3
+ "pins" : [
4
+ {
5
+ "identity" : "commander",
6
+ "kind" : "remoteSourceControl",
7
+ "location" : "https://github.com/steipete/Commander.git",
8
+ "state" : {
9
+ "revision" : "9e349575c8e3c6745e81fe19e5bb5efa01b078ce",
10
+ "version" : "0.2.1"
11
+ }
12
+ },
13
+ {
14
+ "identity" : "elevenlabskit",
15
+ "kind" : "remoteSourceControl",
16
+ "location" : "https://github.com/steipete/ElevenLabsKit",
17
+ "state" : {
18
+ "revision" : "7e3c948d8340abe3977014f3de020edf221e9269",
19
+ "version" : "0.1.0"
20
+ }
21
+ },
22
+ {
23
+ "identity" : "swift-concurrency-extras",
24
+ "kind" : "remoteSourceControl",
25
+ "location" : "https://github.com/pointfreeco/swift-concurrency-extras",
26
+ "state" : {
27
+ "revision" : "5a3825302b1a0d744183200915a47b508c828e6f",
28
+ "version" : "1.3.2"
29
+ }
30
+ },
31
+ {
32
+ "identity" : "swift-syntax",
33
+ "kind" : "remoteSourceControl",
34
+ "location" : "https://github.com/swiftlang/swift-syntax.git",
35
+ "state" : {
36
+ "revision" : "0687f71944021d616d34d922343dcef086855920",
37
+ "version" : "600.0.1"
38
+ }
39
+ },
40
+ {
41
+ "identity" : "swift-testing",
42
+ "kind" : "remoteSourceControl",
43
+ "location" : "https://github.com/apple/swift-testing",
44
+ "state" : {
45
+ "revision" : "399f76dcd91e4c688ca2301fa24a8cc6d9927211",
46
+ "version" : "0.99.0"
47
+ }
48
+ },
49
+ {
50
+ "identity" : "swiftui-math",
51
+ "kind" : "remoteSourceControl",
52
+ "location" : "https://github.com/gonzalezreal/swiftui-math",
53
+ "state" : {
54
+ "revision" : "0b5c2cfaaec8d6193db206f675048eeb5ce95f71",
55
+ "version" : "0.1.0"
56
+ }
57
+ },
58
+ {
59
+ "identity" : "textual",
60
+ "kind" : "remoteSourceControl",
61
+ "location" : "https://github.com/gonzalezreal/textual",
62
+ "state" : {
63
+ "revision" : "5b06b811c0f5313b6b84bbef98c635a630638c38",
64
+ "version" : "0.3.1"
65
+ }
66
+ }
67
+ ],
68
+ "version" : 3
69
+ }
Swabble/Package.swift ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // swift-tools-version: 6.2
2
+ import PackageDescription
3
+
4
+ let package = Package(
5
+ name: "swabble",
6
+ platforms: [
7
+ .macOS(.v15),
8
+ .iOS(.v17),
9
+ ],
10
+ products: [
11
+ .library(name: "Swabble", targets: ["Swabble"]),
12
+ .library(name: "SwabbleKit", targets: ["SwabbleKit"]),
13
+ .executable(name: "swabble", targets: ["SwabbleCLI"]),
14
+ ],
15
+ dependencies: [
16
+ .package(url: "https://github.com/steipete/Commander.git", exact: "0.2.1"),
17
+ .package(url: "https://github.com/apple/swift-testing", from: "0.99.0"),
18
+ ],
19
+ targets: [
20
+ .target(
21
+ name: "Swabble",
22
+ path: "Sources/SwabbleCore",
23
+ swiftSettings: []),
24
+ .target(
25
+ name: "SwabbleKit",
26
+ path: "Sources/SwabbleKit",
27
+ swiftSettings: [
28
+ .enableUpcomingFeature("StrictConcurrency"),
29
+ ]),
30
+ .executableTarget(
31
+ name: "SwabbleCLI",
32
+ dependencies: [
33
+ "Swabble",
34
+ "SwabbleKit",
35
+ .product(name: "Commander", package: "Commander"),
36
+ ],
37
+ path: "Sources/swabble"),
38
+ .testTarget(
39
+ name: "SwabbleKitTests",
40
+ dependencies: [
41
+ "SwabbleKit",
42
+ .product(name: "Testing", package: "swift-testing"),
43
+ ],
44
+ swiftSettings: [
45
+ .enableUpcomingFeature("StrictConcurrency"),
46
+ .enableExperimentalFeature("SwiftTesting"),
47
+ ]),
48
+ .testTarget(
49
+ name: "swabbleTests",
50
+ dependencies: [
51
+ "Swabble",
52
+ .product(name: "Testing", package: "swift-testing"),
53
+ ]),
54
+ ],
55
+ swiftLanguageModes: [.v6])
Swabble/README.md ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🎙️ swabble — Speech.framework wake-word hook daemon (macOS 26)
2
+
3
+ swabble is a Swift 6.2 wake-word hook daemon. The CLI targets macOS 26 (SpeechAnalyzer + SpeechTranscriber). The shared `SwabbleKit` target is multi-platform and exposes wake-word gating utilities for iOS/macOS apps.
4
+
5
+ - **Local-only**: Speech.framework on-device models; zero network usage.
6
+ - **Wake word**: Default `clawd` (aliases `claude`), optional `--no-wake` bypass.
7
+ - **SwabbleKit**: Shared wake gate utilities (gap-based gating when you provide speech segments).
8
+ - **Hooks**: Run any command with prefix/env, cooldown, min_chars, timeout.
9
+ - **Services**: launchd helper stubs for start/stop/install.
10
+ - **File transcribe**: TXT or SRT with time ranges (using AttributedString splits).
11
+
12
+ ## Quick start
13
+ ```bash
14
+ # Install deps
15
+ brew install swiftformat swiftlint
16
+
17
+ # Build
18
+ swift build
19
+
20
+ # Write default config (~/.config/swabble/config.json)
21
+ swift run swabble setup
22
+
23
+ # Run foreground daemon
24
+ swift run swabble serve
25
+
26
+ # Test your hook
27
+ swift run swabble test-hook "hello world"
28
+
29
+ # Transcribe a file to SRT
30
+ swift run swabble transcribe /path/to/audio.m4a --format srt --output out.srt
31
+ ```
32
+
33
+ ## Use as a library
34
+ Add swabble as a SwiftPM dependency and import the `Swabble` or `SwabbleKit` product:
35
+
36
+ ```swift
37
+ // Package.swift
38
+ dependencies: [
39
+ .package(url: "https://github.com/steipete/swabble.git", branch: "main"),
40
+ ],
41
+ targets: [
42
+ .target(name: "MyApp", dependencies: [
43
+ .product(name: "Swabble", package: "swabble"), // Speech pipeline (macOS 26+ / iOS 26+)
44
+ .product(name: "SwabbleKit", package: "swabble"), // Wake-word gate utilities (iOS 17+ / macOS 15+)
45
+ ]),
46
+ ]
47
+ ```
48
+
49
+ ## CLI
50
+ - `serve` — foreground loop (mic → wake → hook)
51
+ - `transcribe <file>` — offline transcription (txt|srt)
52
+ - `test-hook "text"` — invoke configured hook
53
+ - `mic list|set <index>` — enumerate/select input device
54
+ - `setup` — write default config JSON
55
+ - `doctor` — check Speech auth & device availability
56
+ - `health` — prints `ok`
57
+ - `tail-log` — last 10 transcripts
58
+ - `status` — show wake state + recent transcripts
59
+ - `service install|uninstall|status` — user launchd plist (stub: prints launchctl commands)
60
+ - `start|stop|restart` — placeholders until full launchd wiring
61
+
62
+ All commands accept Commander runtime flags (`-v/--verbose`, `--json-output`, `--log-level`), plus `--config` where applicable.
63
+
64
+ ## Config
65
+ `~/.config/swabble/config.json` (auto-created by `setup`):
66
+ ```json
67
+ {
68
+ "audio": {"deviceName": "", "deviceIndex": -1, "sampleRate": 16000, "channels": 1},
69
+ "wake": {"enabled": true, "word": "clawd", "aliases": ["claude"]},
70
+ "hook": {
71
+ "command": "",
72
+ "args": [],
73
+ "prefix": "Voice swabble from ${hostname}: ",
74
+ "cooldownSeconds": 1,
75
+ "minCharacters": 24,
76
+ "timeoutSeconds": 5,
77
+ "env": {}
78
+ },
79
+ "logging": {"level": "info", "format": "text"},
80
+ "transcripts": {"enabled": true, "maxEntries": 50},
81
+ "speech": {"localeIdentifier": "en_US", "etiquetteReplacements": false}
82
+ }
83
+ ```
84
+
85
+ - Config path override: `--config /path/to/config.json` on relevant commands.
86
+ - Transcripts persist to `~/Library/Application Support/swabble/transcripts.log`.
87
+
88
+ ## Hook protocol
89
+ When a wake-gated transcript passes min_chars & cooldown, swabble runs:
90
+ ```
91
+ <command> <args...> "<prefix><text>"
92
+ ```
93
+ Environment variables:
94
+ - `SWABBLE_TEXT` — stripped transcript (wake word removed)
95
+ - `SWABBLE_PREFIX` — rendered prefix (hostname substituted)
96
+ - plus any `hook.env` key/values
97
+
98
+ ## Speech pipeline
99
+ - `AVAudioEngine` tap → `BufferConverter` → `AnalyzerInput` → `SpeechAnalyzer` with a `SpeechTranscriber` module.
100
+ - Requests volatile + final results; the CLI uses text-only wake gating today.
101
+ - Authorization requested at first start; requires macOS 26 + new Speech.framework APIs.
102
+
103
+ ## Development
104
+ - Format: `./scripts/format.sh` (uses local `.swiftformat`)
105
+ - Lint: `./scripts/lint.sh` (uses local `.swiftlint.yml`)
106
+ - Tests: `swift test` (uses swift-testing package)
107
+
108
+ ## Roadmap
109
+ - launchd control (load/bootout, PID + status socket)
110
+ - JSON logging + PII redaction toggle
111
+ - Stronger wake-word detection and control socket status/health
Swabble/Sources/SwabbleCore/Config/Config.swift ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Foundation
2
+
3
+ public struct SwabbleConfig: Codable, Sendable {
4
+ public struct Audio: Codable, Sendable {
5
+ public var deviceName: String = ""
6
+ public var deviceIndex: Int = -1
7
+ public var sampleRate: Double = 16000
8
+ public var channels: Int = 1
9
+ }
10
+
11
+ public struct Wake: Codable, Sendable {
12
+ public var enabled: Bool = true
13
+ public var word: String = "clawd"
14
+ public var aliases: [String] = ["claude"]
15
+ }
16
+
17
+ public struct Hook: Codable, Sendable {
18
+ public var command: String = ""
19
+ public var args: [String] = []
20
+ public var prefix: String = "Voice swabble from ${hostname}: "
21
+ public var cooldownSeconds: Double = 1
22
+ public var minCharacters: Int = 24
23
+ public var timeoutSeconds: Double = 5
24
+ public var env: [String: String] = [:]
25
+ }
26
+
27
+ public struct Logging: Codable, Sendable {
28
+ public var level: String = "info"
29
+ public var format: String = "text" // text|json placeholder
30
+ }
31
+
32
+ public struct Transcripts: Codable, Sendable {
33
+ public var enabled: Bool = true
34
+ public var maxEntries: Int = 50
35
+ }
36
+
37
+ public struct Speech: Codable, Sendable {
38
+ public var localeIdentifier: String = Locale.current.identifier
39
+ public var etiquetteReplacements: Bool = false
40
+ }
41
+
42
+ public var audio = Audio()
43
+ public var wake = Wake()
44
+ public var hook = Hook()
45
+ public var logging = Logging()
46
+ public var transcripts = Transcripts()
47
+ public var speech = Speech()
48
+
49
+ public static let defaultPath = FileManager.default
50
+ .homeDirectoryForCurrentUser
51
+ .appendingPathComponent(".config/swabble/config.json")
52
+
53
+ public init() {}
54
+ }
55
+
56
+ public enum ConfigError: Error {
57
+ case missingConfig
58
+ }
59
+
60
+ public enum ConfigLoader {
61
+ public static func load(at path: URL?) throws -> SwabbleConfig {
62
+ let url = path ?? SwabbleConfig.defaultPath
63
+ if !FileManager.default.fileExists(atPath: url.path) {
64
+ throw ConfigError.missingConfig
65
+ }
66
+ let data = try Data(contentsOf: url)
67
+ return try JSONDecoder().decode(SwabbleConfig.self, from: data)
68
+ }
69
+
70
+ public static func save(_ config: SwabbleConfig, at path: URL?) throws {
71
+ let url = path ?? SwabbleConfig.defaultPath
72
+ let dir = url.deletingLastPathComponent()
73
+ try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
74
+ let data = try JSONEncoder().encode(config)
75
+ try data.write(to: url)
76
+ }
77
+ }
Swabble/Sources/SwabbleCore/Hooks/HookExecutor.swift ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Foundation
2
+
3
+ public struct HookJob: Sendable {
4
+ public let text: String
5
+ public let timestamp: Date
6
+
7
+ public init(text: String, timestamp: Date) {
8
+ self.text = text
9
+ self.timestamp = timestamp
10
+ }
11
+ }
12
+
13
+ public actor HookExecutor {
14
+ private let config: SwabbleConfig
15
+ private var lastRun: Date?
16
+ private let hostname: String
17
+
18
+ public init(config: SwabbleConfig) {
19
+ self.config = config
20
+ hostname = Host.current().localizedName ?? "host"
21
+ }
22
+
23
+ public func shouldRun() -> Bool {
24
+ guard config.hook.cooldownSeconds > 0 else { return true }
25
+ if let lastRun, Date().timeIntervalSince(lastRun) < config.hook.cooldownSeconds {
26
+ return false
27
+ }
28
+ return true
29
+ }
30
+
31
+ public func run(job: HookJob) async throws {
32
+ guard shouldRun() else { return }
33
+ guard !config.hook.command.isEmpty else { throw NSError(
34
+ domain: "Hook",
35
+ code: 1,
36
+ userInfo: [NSLocalizedDescriptionKey: "hook command not set"]) }
37
+
38
+ let prefix = config.hook.prefix.replacingOccurrences(of: "${hostname}", with: hostname)
39
+ let payload = prefix + job.text
40
+
41
+ let process = Process()
42
+ process.executableURL = URL(fileURLWithPath: config.hook.command)
43
+ process.arguments = config.hook.args + [payload]
44
+
45
+ var env = ProcessInfo.processInfo.environment
46
+ env["SWABBLE_TEXT"] = job.text
47
+ env["SWABBLE_PREFIX"] = prefix
48
+ for (k, v) in config.hook.env {
49
+ env[k] = v
50
+ }
51
+ process.environment = env
52
+
53
+ let pipe = Pipe()
54
+ process.standardOutput = pipe
55
+ process.standardError = pipe
56
+
57
+ try process.run()
58
+
59
+ let timeoutNanos = UInt64(max(config.hook.timeoutSeconds, 0.1) * 1_000_000_000)
60
+ try await withThrowingTaskGroup(of: Void.self) { group in
61
+ group.addTask {
62
+ process.waitUntilExit()
63
+ }
64
+ group.addTask {
65
+ try await Task.sleep(nanoseconds: timeoutNanos)
66
+ if process.isRunning {
67
+ process.terminate()
68
+ }
69
+ }
70
+ try await group.next()
71
+ group.cancelAll()
72
+ }
73
+ lastRun = Date()
74
+ }
75
+ }
Swabble/Sources/SwabbleCore/Speech/BufferConverter.swift ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ @preconcurrency import AVFoundation
2
+ import Foundation
3
+
4
+ final class BufferConverter {
5
+ private final class Box<T>: @unchecked Sendable { var value: T; init(_ value: T) { self.value = value } }
6
+ enum ConverterError: Swift.Error {
7
+ case failedToCreateConverter
8
+ case failedToCreateConversionBuffer
9
+ case conversionFailed(NSError?)
10
+ }
11
+
12
+ private var converter: AVAudioConverter?
13
+
14
+ func convert(_ buffer: AVAudioPCMBuffer, to format: AVAudioFormat) throws -> AVAudioPCMBuffer {
15
+ let inputFormat = buffer.format
16
+ if inputFormat == format {
17
+ return buffer
18
+ }
19
+ if converter == nil || converter?.outputFormat != format {
20
+ converter = AVAudioConverter(from: inputFormat, to: format)
21
+ converter?.primeMethod = .none
22
+ }
23
+ guard let converter else { throw ConverterError.failedToCreateConverter }
24
+
25
+ let sampleRateRatio = converter.outputFormat.sampleRate / converter.inputFormat.sampleRate
26
+ let scaledInputFrameLength = Double(buffer.frameLength) * sampleRateRatio
27
+ let frameCapacity = AVAudioFrameCount(scaledInputFrameLength.rounded(.up))
28
+ guard let conversionBuffer = AVAudioPCMBuffer(pcmFormat: converter.outputFormat, frameCapacity: frameCapacity)
29
+ else {
30
+ throw ConverterError.failedToCreateConversionBuffer
31
+ }
32
+
33
+ var nsError: NSError?
34
+ let consumed = Box(false)
35
+ let inputBuffer = buffer
36
+ let status = converter.convert(to: conversionBuffer, error: &nsError) { _, statusPtr in
37
+ if consumed.value {
38
+ statusPtr.pointee = .noDataNow
39
+ return nil
40
+ }
41
+ consumed.value = true
42
+ statusPtr.pointee = .haveData
43
+ return inputBuffer
44
+ }
45
+ if status == .error {
46
+ throw ConverterError.conversionFailed(nsError)
47
+ }
48
+ return conversionBuffer
49
+ }
50
+ }
Swabble/Sources/SwabbleCore/Speech/SpeechPipeline.swift ADDED
@@ -0,0 +1,114 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import AVFoundation
2
+ import Foundation
3
+ import Speech
4
+
5
+ @available(macOS 26.0, iOS 26.0, *)
6
+ public struct SpeechSegment: Sendable {
7
+ public let text: String
8
+ public let isFinal: Bool
9
+ }
10
+
11
+ @available(macOS 26.0, iOS 26.0, *)
12
+ public enum SpeechPipelineError: Error {
13
+ case authorizationDenied
14
+ case analyzerFormatUnavailable
15
+ case transcriberUnavailable
16
+ }
17
+
18
+ /// Live microphone → SpeechAnalyzer → SpeechTranscriber pipeline.
19
+ @available(macOS 26.0, iOS 26.0, *)
20
+ public actor SpeechPipeline {
21
+ private struct UnsafeBuffer: @unchecked Sendable { let buffer: AVAudioPCMBuffer }
22
+
23
+ private var engine = AVAudioEngine()
24
+ private var transcriber: SpeechTranscriber?
25
+ private var analyzer: SpeechAnalyzer?
26
+ private var inputContinuation: AsyncStream<AnalyzerInput>.Continuation?
27
+ private var resultTask: Task<Void, Never>?
28
+ private let converter = BufferConverter()
29
+
30
+ public init() {}
31
+
32
+ public func start(localeIdentifier: String, etiquette: Bool) async throws -> AsyncStream<SpeechSegment> {
33
+ let auth = await requestAuthorizationIfNeeded()
34
+ guard auth == .authorized else { throw SpeechPipelineError.authorizationDenied }
35
+
36
+ let transcriberModule = SpeechTranscriber(
37
+ locale: Locale(identifier: localeIdentifier),
38
+ transcriptionOptions: etiquette ? [.etiquetteReplacements] : [],
39
+ reportingOptions: [.volatileResults],
40
+ attributeOptions: [])
41
+ transcriber = transcriberModule
42
+
43
+ guard let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriberModule])
44
+ else {
45
+ throw SpeechPipelineError.analyzerFormatUnavailable
46
+ }
47
+
48
+ analyzer = SpeechAnalyzer(modules: [transcriberModule])
49
+ let (stream, continuation) = AsyncStream<AnalyzerInput>.makeStream()
50
+ inputContinuation = continuation
51
+
52
+ let inputNode = engine.inputNode
53
+ let inputFormat = inputNode.outputFormat(forBus: 0)
54
+ inputNode.removeTap(onBus: 0)
55
+ inputNode.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { [weak self] buffer, _ in
56
+ guard let self else { return }
57
+ let boxed = UnsafeBuffer(buffer: buffer)
58
+ Task { await self.handleBuffer(boxed.buffer, targetFormat: analyzerFormat) }
59
+ }
60
+
61
+ engine.prepare()
62
+ try engine.start()
63
+ try await analyzer?.start(inputSequence: stream)
64
+
65
+ guard let transcriberForStream = transcriber else {
66
+ throw SpeechPipelineError.transcriberUnavailable
67
+ }
68
+
69
+ return AsyncStream { continuation in
70
+ self.resultTask = Task {
71
+ do {
72
+ for try await result in transcriberForStream.results {
73
+ let seg = SpeechSegment(text: String(result.text.characters), isFinal: result.isFinal)
74
+ continuation.yield(seg)
75
+ }
76
+ } catch {
77
+ // swallow errors and finish
78
+ }
79
+ continuation.finish()
80
+ }
81
+ continuation.onTermination = { _ in
82
+ Task { await self.stop() }
83
+ }
84
+ }
85
+ }
86
+
87
+ public func stop() async {
88
+ resultTask?.cancel()
89
+ inputContinuation?.finish()
90
+ engine.inputNode.removeTap(onBus: 0)
91
+ engine.stop()
92
+ try? await analyzer?.finalizeAndFinishThroughEndOfInput()
93
+ }
94
+
95
+ private func handleBuffer(_ buffer: AVAudioPCMBuffer, targetFormat: AVAudioFormat) async {
96
+ do {
97
+ let converted = try converter.convert(buffer, to: targetFormat)
98
+ let input = AnalyzerInput(buffer: converted)
99
+ inputContinuation?.yield(input)
100
+ } catch {
101
+ // drop on conversion failure
102
+ }
103
+ }
104
+
105
+ private func requestAuthorizationIfNeeded() async -> SFSpeechRecognizerAuthorizationStatus {
106
+ let current = SFSpeechRecognizer.authorizationStatus()
107
+ guard current == .notDetermined else { return current }
108
+ return await withCheckedContinuation { continuation in
109
+ SFSpeechRecognizer.requestAuthorization { status in
110
+ continuation.resume(returning: status)
111
+ }
112
+ }
113
+ }
114
+ }
Swabble/Sources/SwabbleCore/Support/AttributedString+Sentences.swift ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import CoreMedia
2
+ import Foundation
3
+ import NaturalLanguage
4
+
5
+ extension AttributedString {
6
+ public func sentences(maxLength: Int? = nil) -> [AttributedString] {
7
+ let tokenizer = NLTokenizer(unit: .sentence)
8
+ let string = String(characters)
9
+ tokenizer.string = string
10
+ let sentenceRanges = tokenizer.tokens(for: string.startIndex..<string.endIndex).map {
11
+ (
12
+ $0,
13
+ AttributedString.Index($0.lowerBound, within: self)!
14
+ ..<
15
+ AttributedString.Index($0.upperBound, within: self)!)
16
+ }
17
+ let ranges = sentenceRanges.flatMap { sentenceStringRange, sentenceRange in
18
+ let sentence = self[sentenceRange]
19
+ guard let maxLength, sentence.characters.count > maxLength else {
20
+ return [sentenceRange]
21
+ }
22
+
23
+ let wordTokenizer = NLTokenizer(unit: .word)
24
+ wordTokenizer.string = string
25
+ var wordRanges = wordTokenizer.tokens(for: sentenceStringRange).map {
26
+ AttributedString.Index($0.lowerBound, within: self)!
27
+ ..<
28
+ AttributedString.Index($0.upperBound, within: self)!
29
+ }
30
+ guard !wordRanges.isEmpty else { return [sentenceRange] }
31
+ wordRanges[0] = sentenceRange.lowerBound..<wordRanges[0].upperBound
32
+ wordRanges[wordRanges.count - 1] = wordRanges[wordRanges.count - 1].lowerBound..<sentenceRange.upperBound
33
+
34
+ var ranges: [Range<AttributedString.Index>] = []
35
+ for wordRange in wordRanges {
36
+ if let lastRange = ranges.last,
37
+ self[lastRange].characters.count + self[wordRange].characters.count <= maxLength {
38
+ ranges[ranges.count - 1] = lastRange.lowerBound..<wordRange.upperBound
39
+ } else {
40
+ ranges.append(wordRange)
41
+ }
42
+ }
43
+
44
+ return ranges
45
+ }
46
+
47
+ return ranges.compactMap { range in
48
+ let audioTimeRanges = self[range].runs.filter {
49
+ !String(self[$0.range].characters)
50
+ .trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
51
+ }.compactMap(\.audioTimeRange)
52
+ guard !audioTimeRanges.isEmpty else { return nil }
53
+ let start = audioTimeRanges.first!.start
54
+ let end = audioTimeRanges.last!.end
55
+ var attributes = AttributeContainer()
56
+ attributes[AttributeScopes.SpeechAttributes.TimeRangeAttribute.self] = CMTimeRange(
57
+ start: start,
58
+ end: end)
59
+ return AttributedString(self[range].characters, attributes: attributes)
60
+ }
61
+ }
62
+ }
Swabble/Sources/SwabbleCore/Support/Logging.swift ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Foundation
2
+
3
+ public enum LogLevel: String, Comparable, CaseIterable, Sendable {
4
+ case trace, debug, info, warn, error
5
+
6
+ var rank: Int {
7
+ switch self {
8
+ case .trace: 0
9
+ case .debug: 1
10
+ case .info: 2
11
+ case .warn: 3
12
+ case .error: 4
13
+ }
14
+ }
15
+
16
+ public static func < (lhs: LogLevel, rhs: LogLevel) -> Bool { lhs.rank < rhs.rank }
17
+ }
18
+
19
+ public struct Logger: Sendable {
20
+ public let level: LogLevel
21
+
22
+ public init(level: LogLevel) { self.level = level }
23
+
24
+ public func log(_ level: LogLevel, _ message: String) {
25
+ guard level >= self.level else { return }
26
+ let ts = ISO8601DateFormatter().string(from: Date())
27
+ print("[\(level.rawValue.uppercased())] \(ts) | \(message)")
28
+ }
29
+
30
+ public func trace(_ msg: String) { log(.trace, msg) }
31
+ public func debug(_ msg: String) { log(.debug, msg) }
32
+ public func info(_ msg: String) { log(.info, msg) }
33
+ public func warn(_ msg: String) { log(.warn, msg) }
34
+ public func error(_ msg: String) { log(.error, msg) }
35
+ }
36
+
37
+ extension LogLevel {
38
+ public init?(configValue: String) {
39
+ self.init(rawValue: configValue.lowercased())
40
+ }
41
+ }
Swabble/Sources/SwabbleCore/Support/OutputFormat.swift ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import CoreMedia
2
+ import Foundation
3
+
4
+ public enum OutputFormat: String {
5
+ case txt
6
+ case srt
7
+
8
+ public var needsAudioTimeRange: Bool {
9
+ switch self {
10
+ case .srt: true
11
+ default: false
12
+ }
13
+ }
14
+
15
+ public func text(for transcript: AttributedString, maxLength: Int) -> String {
16
+ switch self {
17
+ case .txt:
18
+ return String(transcript.characters)
19
+ case .srt:
20
+ func format(_ timeInterval: TimeInterval) -> String {
21
+ let ms = Int(timeInterval.truncatingRemainder(dividingBy: 1) * 1000)
22
+ let s = Int(timeInterval) % 60
23
+ let m = (Int(timeInterval) / 60) % 60
24
+ let h = Int(timeInterval) / 60 / 60
25
+ return String(format: "%0.2d:%0.2d:%0.2d,%0.3d", h, m, s, ms)
26
+ }
27
+
28
+ return transcript.sentences(maxLength: maxLength).compactMap { (sentence: AttributedString) -> (
29
+ CMTimeRange,
30
+ String)? in
31
+ guard let timeRange = sentence.audioTimeRange else { return nil }
32
+ return (timeRange, String(sentence.characters))
33
+ }.enumerated().map { index, run in
34
+ let (timeRange, text) = run
35
+ return """
36
+
37
+ \(index + 1)
38
+ \(format(timeRange.start.seconds)) --> \(format(timeRange.end.seconds))
39
+ \(text.trimmingCharacters(in: .whitespacesAndNewlines))
40
+
41
+ """
42
+ }.joined().trimmingCharacters(in: .whitespacesAndNewlines)
43
+ }
44
+ }
45
+ }
Swabble/Sources/SwabbleCore/Support/TranscriptsStore.swift ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Foundation
2
+
3
+ public actor TranscriptsStore {
4
+ public static let shared = TranscriptsStore()
5
+
6
+ private var entries: [String] = []
7
+ private let limit = 100
8
+ private let fileURL: URL
9
+
10
+ public init() {
11
+ let dir = FileManager.default.homeDirectoryForCurrentUser
12
+ .appendingPathComponent("Library/Application Support/swabble", isDirectory: true)
13
+ try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
14
+ fileURL = dir.appendingPathComponent("transcripts.log")
15
+ if let data = try? Data(contentsOf: fileURL),
16
+ let text = String(data: data, encoding: .utf8) {
17
+ entries = text.split(separator: "\n").map(String.init).suffix(limit)
18
+ }
19
+ }
20
+
21
+ public func append(text: String) {
22
+ entries.append(text)
23
+ if entries.count > limit {
24
+ entries.removeFirst(entries.count - limit)
25
+ }
26
+ let body = entries.joined(separator: "\n")
27
+ try? body.write(to: fileURL, atomically: false, encoding: .utf8)
28
+ }
29
+
30
+ public func latest() -> [String] { entries }
31
+ }
32
+
33
+ extension String {
34
+ private func appendLine(to url: URL) throws {
35
+ let data = (self + "\n").data(using: .utf8) ?? Data()
36
+ if FileManager.default.fileExists(atPath: url.path) {
37
+ let handle = try FileHandle(forWritingTo: url)
38
+ try handle.seekToEnd()
39
+ try handle.write(contentsOf: data)
40
+ try handle.close()
41
+ } else {
42
+ try data.write(to: url)
43
+ }
44
+ }
45
+ }
Swabble/Sources/SwabbleKit/WakeWordGate.swift ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Foundation
2
+
3
+ public struct WakeWordSegment: Sendable, Equatable {
4
+ public let text: String
5
+ public let start: TimeInterval
6
+ public let duration: TimeInterval
7
+ public let range: Range<String.Index>?
8
+
9
+ public init(text: String, start: TimeInterval, duration: TimeInterval, range: Range<String.Index>? = nil) {
10
+ self.text = text
11
+ self.start = start
12
+ self.duration = duration
13
+ self.range = range
14
+ }
15
+
16
+ public var end: TimeInterval { start + duration }
17
+ }
18
+
19
+ public struct WakeWordGateConfig: Sendable, Equatable {
20
+ public var triggers: [String]
21
+ public var minPostTriggerGap: TimeInterval
22
+ public var minCommandLength: Int
23
+
24
+ public init(
25
+ triggers: [String],
26
+ minPostTriggerGap: TimeInterval = 0.45,
27
+ minCommandLength: Int = 1) {
28
+ self.triggers = triggers
29
+ self.minPostTriggerGap = minPostTriggerGap
30
+ self.minCommandLength = minCommandLength
31
+ }
32
+ }
33
+
34
+ public struct WakeWordGateMatch: Sendable, Equatable {
35
+ public let triggerEndTime: TimeInterval
36
+ public let postGap: TimeInterval
37
+ public let command: String
38
+
39
+ public init(triggerEndTime: TimeInterval, postGap: TimeInterval, command: String) {
40
+ self.triggerEndTime = triggerEndTime
41
+ self.postGap = postGap
42
+ self.command = command
43
+ }
44
+ }
45
+
46
+ public enum WakeWordGate {
47
+ private struct Token {
48
+ let normalized: String
49
+ let start: TimeInterval
50
+ let end: TimeInterval
51
+ let range: Range<String.Index>?
52
+ let text: String
53
+ }
54
+
55
+ private struct TriggerTokens {
56
+ let tokens: [String]
57
+ }
58
+
59
+ private struct MatchCandidate {
60
+ let index: Int
61
+ let triggerEnd: TimeInterval
62
+ let gap: TimeInterval
63
+ }
64
+
65
+ public static func match(
66
+ transcript: String,
67
+ segments: [WakeWordSegment],
68
+ config: WakeWordGateConfig)
69
+ -> WakeWordGateMatch? {
70
+ let triggerTokens = normalizeTriggers(config.triggers)
71
+ guard !triggerTokens.isEmpty else { return nil }
72
+
73
+ let tokens = normalizeSegments(segments)
74
+ guard !tokens.isEmpty else { return nil }
75
+
76
+ var best: MatchCandidate?
77
+
78
+ for trigger in triggerTokens {
79
+ let count = trigger.tokens.count
80
+ guard count > 0, tokens.count > count else { continue }
81
+ for i in 0...(tokens.count - count - 1) {
82
+ let matched = (0..<count).allSatisfy { tokens[i + $0].normalized == trigger.tokens[$0] }
83
+ if !matched { continue }
84
+
85
+ let triggerEnd = tokens[i + count - 1].end
86
+ let nextToken = tokens[i + count]
87
+ let gap = nextToken.start - triggerEnd
88
+ if gap < config.minPostTriggerGap { continue }
89
+
90
+ if let best, i <= best.index { continue }
91
+
92
+ best = MatchCandidate(index: i, triggerEnd: triggerEnd, gap: gap)
93
+ }
94
+ }
95
+
96
+ guard let best else { return nil }
97
+ let command = commandText(transcript: transcript, segments: segments, triggerEndTime: best.triggerEnd)
98
+ .trimmingCharacters(in: Self.whitespaceAndPunctuation)
99
+ guard command.count >= config.minCommandLength else { return nil }
100
+ return WakeWordGateMatch(triggerEndTime: best.triggerEnd, postGap: best.gap, command: command)
101
+ }
102
+
103
+ public static func commandText(
104
+ transcript: String,
105
+ segments: [WakeWordSegment],
106
+ triggerEndTime: TimeInterval)
107
+ -> String {
108
+ let threshold = triggerEndTime + 0.001
109
+ for segment in segments where segment.start >= threshold {
110
+ if normalizeToken(segment.text).isEmpty { continue }
111
+ if let range = segment.range {
112
+ let slice = transcript[range.lowerBound...]
113
+ return String(slice).trimmingCharacters(in: Self.whitespaceAndPunctuation)
114
+ }
115
+ break
116
+ }
117
+
118
+ let text = segments
119
+ .filter { $0.start >= threshold && !normalizeToken($0.text).isEmpty }
120
+ .map(\.text)
121
+ .joined(separator: " ")
122
+ return text.trimmingCharacters(in: Self.whitespaceAndPunctuation)
123
+ }
124
+
125
+ public static func matchesTextOnly(text: String, triggers: [String]) -> Bool {
126
+ guard !text.isEmpty else { return false }
127
+ let normalized = text.lowercased()
128
+ for trigger in triggers {
129
+ let token = trigger.trimmingCharacters(in: whitespaceAndPunctuation).lowercased()
130
+ if token.isEmpty { continue }
131
+ if normalized.contains(token) { return true }
132
+ }
133
+ return false
134
+ }
135
+
136
+ public static func stripWake(text: String, triggers: [String]) -> String {
137
+ var out = text
138
+ for trigger in triggers {
139
+ let token = trigger.trimmingCharacters(in: whitespaceAndPunctuation)
140
+ guard !token.isEmpty else { continue }
141
+ out = out.replacingOccurrences(of: token, with: "", options: [.caseInsensitive])
142
+ }
143
+ return out.trimmingCharacters(in: whitespaceAndPunctuation)
144
+ }
145
+
146
+ private static func normalizeTriggers(_ triggers: [String]) -> [TriggerTokens] {
147
+ var output: [TriggerTokens] = []
148
+ for trigger in triggers {
149
+ let tokens = trigger
150
+ .split(whereSeparator: { $0.isWhitespace })
151
+ .map { normalizeToken(String($0)) }
152
+ .filter { !$0.isEmpty }
153
+ if tokens.isEmpty { continue }
154
+ output.append(TriggerTokens(tokens: tokens))
155
+ }
156
+ return output
157
+ }
158
+
159
+ private static func normalizeSegments(_ segments: [WakeWordSegment]) -> [Token] {
160
+ segments.compactMap { segment in
161
+ let normalized = normalizeToken(segment.text)
162
+ guard !normalized.isEmpty else { return nil }
163
+ return Token(
164
+ normalized: normalized,
165
+ start: segment.start,
166
+ end: segment.end,
167
+ range: segment.range,
168
+ text: segment.text)
169
+ }
170
+ }
171
+
172
+ private static func normalizeToken(_ token: String) -> String {
173
+ token
174
+ .trimmingCharacters(in: whitespaceAndPunctuation)
175
+ .lowercased()
176
+ }
177
+
178
+ private static let whitespaceAndPunctuation = CharacterSet.whitespacesAndNewlines
179
+ .union(.punctuationCharacters)
180
+ }
181
+
182
+ #if canImport(Speech)
183
+ import Speech
184
+
185
+ public enum WakeWordSpeechSegments {
186
+ public static func from(transcription: SFTranscription, transcript: String) -> [WakeWordSegment] {
187
+ transcription.segments.map { segment in
188
+ let range = Range(segment.substringRange, in: transcript)
189
+ return WakeWordSegment(
190
+ text: segment.substring,
191
+ start: segment.timestamp,
192
+ duration: segment.duration,
193
+ range: range)
194
+ }
195
+ }
196
+ }
197
+ #endif
Swabble/Sources/swabble/CLI/CLIRegistry.swift ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+
4
+ @available(macOS 26.0, *)
5
+ @MainActor
6
+ enum CLIRegistry {
7
+ static var descriptors: [CommandDescriptor] {
8
+ let serveDesc = descriptor(for: ServeCommand.self)
9
+ let transcribeDesc = descriptor(for: TranscribeCommand.self)
10
+ let testHookDesc = descriptor(for: TestHookCommand.self)
11
+ let micList = descriptor(for: MicList.self)
12
+ let micSet = descriptor(for: MicSet.self)
13
+ let micRoot = CommandDescriptor(
14
+ name: "mic",
15
+ abstract: "Microphone management",
16
+ discussion: nil,
17
+ signature: CommandSignature(),
18
+ subcommands: [micList, micSet])
19
+ let serviceRoot = CommandDescriptor(
20
+ name: "service",
21
+ abstract: "launchd helper",
22
+ discussion: nil,
23
+ signature: CommandSignature(),
24
+ subcommands: [
25
+ descriptor(for: ServiceInstall.self),
26
+ descriptor(for: ServiceUninstall.self),
27
+ descriptor(for: ServiceStatus.self)
28
+ ])
29
+ let doctorDesc = descriptor(for: DoctorCommand.self)
30
+ let setupDesc = descriptor(for: SetupCommand.self)
31
+ let healthDesc = descriptor(for: HealthCommand.self)
32
+ let tailLogDesc = descriptor(for: TailLogCommand.self)
33
+ let startDesc = descriptor(for: StartCommand.self)
34
+ let stopDesc = descriptor(for: StopCommand.self)
35
+ let restartDesc = descriptor(for: RestartCommand.self)
36
+ let statusDesc = descriptor(for: StatusCommand.self)
37
+
38
+ let rootSignature = CommandSignature().withStandardRuntimeFlags()
39
+ let root = CommandDescriptor(
40
+ name: "swabble",
41
+ abstract: "Speech hook daemon",
42
+ discussion: "Local wake-word → SpeechTranscriber → hook",
43
+ signature: rootSignature,
44
+ subcommands: [
45
+ serveDesc,
46
+ transcribeDesc,
47
+ testHookDesc,
48
+ micRoot,
49
+ serviceRoot,
50
+ doctorDesc,
51
+ setupDesc,
52
+ healthDesc,
53
+ tailLogDesc,
54
+ startDesc,
55
+ stopDesc,
56
+ restartDesc,
57
+ statusDesc
58
+ ])
59
+ return [root]
60
+ }
61
+
62
+ private static func descriptor(for type: any ParsableCommand.Type) -> CommandDescriptor {
63
+ let sig = CommandSignature.describe(type.init()).withStandardRuntimeFlags()
64
+ return CommandDescriptor(
65
+ name: type.commandDescription.commandName ?? "",
66
+ abstract: type.commandDescription.abstract,
67
+ discussion: type.commandDescription.discussion,
68
+ signature: sig,
69
+ subcommands: [])
70
+ }
71
+ }
Swabble/Sources/swabble/Commands/DoctorCommand.swift ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+ import Speech
4
+ import Swabble
5
+
6
+ @MainActor
7
+ struct DoctorCommand: ParsableCommand {
8
+ static var commandDescription: CommandDescription {
9
+ CommandDescription(commandName: "doctor", abstract: "Check Speech permission and config")
10
+ }
11
+
12
+ @Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
13
+
14
+ init() {}
15
+ init(parsed: ParsedValues) {
16
+ self.init()
17
+ if let cfg = parsed.options["config"]?.last { configPath = cfg }
18
+ }
19
+
20
+ mutating func run() async throws {
21
+ let auth = await SFSpeechRecognizer.authorizationStatus()
22
+ print("Speech auth: \(auth)")
23
+ do {
24
+ _ = try ConfigLoader.load(at: configURL)
25
+ print("Config: OK")
26
+ } catch {
27
+ print("Config missing or invalid; run setup")
28
+ }
29
+ let session = AVCaptureDevice.DiscoverySession(
30
+ deviceTypes: [.microphone, .external],
31
+ mediaType: .audio,
32
+ position: .unspecified)
33
+ print("Mics found: \(session.devices.count)")
34
+ }
35
+
36
+ private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
37
+ }
Swabble/Sources/swabble/Commands/HealthCommand.swift ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+
4
+ @MainActor
5
+ struct HealthCommand: ParsableCommand {
6
+ static var commandDescription: CommandDescription {
7
+ CommandDescription(commandName: "health", abstract: "Health probe")
8
+ }
9
+
10
+ init() {}
11
+ init(parsed: ParsedValues) {}
12
+
13
+ mutating func run() async throws {
14
+ print("ok")
15
+ }
16
+ }
Swabble/Sources/swabble/Commands/MicCommands.swift ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import AVFoundation
2
+ import Commander
3
+ import Foundation
4
+ import Swabble
5
+
6
+ @MainActor
7
+ struct MicCommand: ParsableCommand {
8
+ static var commandDescription: CommandDescription {
9
+ CommandDescription(
10
+ commandName: "mic",
11
+ abstract: "Microphone management",
12
+ subcommands: [MicList.self, MicSet.self])
13
+ }
14
+ }
15
+
16
+ @MainActor
17
+ struct MicList: ParsableCommand {
18
+ static var commandDescription: CommandDescription {
19
+ CommandDescription(commandName: "list", abstract: "List input devices")
20
+ }
21
+
22
+ init() {}
23
+ init(parsed: ParsedValues) {}
24
+
25
+ mutating func run() async throws {
26
+ let session = AVCaptureDevice.DiscoverySession(
27
+ deviceTypes: [.microphone, .external],
28
+ mediaType: .audio,
29
+ position: .unspecified)
30
+ let devices = session.devices
31
+ if devices.isEmpty { print("no audio inputs found"); return }
32
+ for (idx, device) in devices.enumerated() {
33
+ print("[\(idx)] \(device.localizedName)")
34
+ }
35
+ }
36
+ }
37
+
38
+ @MainActor
39
+ struct MicSet: ParsableCommand {
40
+ @Argument(help: "Device index from list") var index: Int = 0
41
+ @Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
42
+
43
+ static var commandDescription: CommandDescription {
44
+ CommandDescription(commandName: "set", abstract: "Set default input device index")
45
+ }
46
+
47
+ init() {}
48
+ init(parsed: ParsedValues) {
49
+ self.init()
50
+ if let value = parsed.positional.first, let intVal = Int(value) { index = intVal }
51
+ if let cfg = parsed.options["config"]?.last { configPath = cfg }
52
+ }
53
+
54
+ mutating func run() async throws {
55
+ var cfg = try ConfigLoader.load(at: configURL)
56
+ cfg.audio.deviceIndex = index
57
+ try ConfigLoader.save(cfg, at: configURL)
58
+ print("saved device index \(index)")
59
+ }
60
+
61
+ private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
62
+ }
Swabble/Sources/swabble/Commands/ServeCommand.swift ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+ import Swabble
4
+ import SwabbleKit
5
+
6
+ @available(macOS 26.0, *)
7
+ @MainActor
8
+ struct ServeCommand: ParsableCommand {
9
+ @Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
10
+ @Flag(name: .long("no-wake"), help: "Disable wake word") var noWake: Bool = false
11
+
12
+ static var commandDescription: CommandDescription {
13
+ CommandDescription(
14
+ commandName: "serve",
15
+ abstract: "Run swabble in the foreground")
16
+ }
17
+
18
+ init() {}
19
+
20
+ init(parsed: ParsedValues) {
21
+ self.init()
22
+ if parsed.flags.contains("noWake") { noWake = true }
23
+ if let cfg = parsed.options["config"]?.last { configPath = cfg }
24
+ }
25
+
26
+ mutating func run() async throws {
27
+ var cfg: SwabbleConfig
28
+ do {
29
+ cfg = try ConfigLoader.load(at: configURL)
30
+ } catch {
31
+ cfg = SwabbleConfig()
32
+ try ConfigLoader.save(cfg, at: configURL)
33
+ }
34
+ if noWake {
35
+ cfg.wake.enabled = false
36
+ }
37
+
38
+ let logger = Logger(level: LogLevel(configValue: cfg.logging.level) ?? .info)
39
+ logger.info("swabble serve starting (wake: \(cfg.wake.enabled ? cfg.wake.word : "disabled"))")
40
+ let pipeline = SpeechPipeline()
41
+ do {
42
+ let stream = try await pipeline.start(
43
+ localeIdentifier: cfg.speech.localeIdentifier,
44
+ etiquette: cfg.speech.etiquetteReplacements)
45
+ for await seg in stream {
46
+ if cfg.wake.enabled {
47
+ guard Self.matchesWake(text: seg.text, cfg: cfg) else { continue }
48
+ }
49
+ let stripped = Self.stripWake(text: seg.text, cfg: cfg)
50
+ let job = HookJob(text: stripped, timestamp: Date())
51
+ let executor = HookExecutor(config: cfg)
52
+ try await executor.run(job: job)
53
+ if cfg.transcripts.enabled {
54
+ await TranscriptsStore.shared.append(text: stripped)
55
+ }
56
+ if seg.isFinal {
57
+ logger.info("final: \(stripped)")
58
+ } else {
59
+ logger.debug("partial: \(stripped)")
60
+ }
61
+ }
62
+ } catch {
63
+ logger.error("serve error: \(error)")
64
+ throw error
65
+ }
66
+ }
67
+
68
+ private var configURL: URL? {
69
+ configPath.map { URL(fileURLWithPath: $0) }
70
+ }
71
+
72
+ private static func matchesWake(text: String, cfg: SwabbleConfig) -> Bool {
73
+ let triggers = [cfg.wake.word] + cfg.wake.aliases
74
+ return WakeWordGate.matchesTextOnly(text: text, triggers: triggers)
75
+ }
76
+
77
+ private static func stripWake(text: String, cfg: SwabbleConfig) -> String {
78
+ let triggers = [cfg.wake.word] + cfg.wake.aliases
79
+ return WakeWordGate.stripWake(text: text, triggers: triggers)
80
+ }
81
+ }
Swabble/Sources/swabble/Commands/ServiceCommands.swift ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+
4
+ @MainActor
5
+ struct ServiceRootCommand: ParsableCommand {
6
+ static var commandDescription: CommandDescription {
7
+ CommandDescription(
8
+ commandName: "service",
9
+ abstract: "Manage launchd agent",
10
+ subcommands: [ServiceInstall.self, ServiceUninstall.self, ServiceStatus.self])
11
+ }
12
+ }
13
+
14
+ private enum LaunchdHelper {
15
+ static let label = "com.swabble.agent"
16
+
17
+ static var plistURL: URL {
18
+ FileManager.default
19
+ .homeDirectoryForCurrentUser
20
+ .appendingPathComponent("Library/LaunchAgents/\(label).plist")
21
+ }
22
+
23
+ static func writePlist(executable: String) throws {
24
+ let plist: [String: Any] = [
25
+ "Label": label,
26
+ "ProgramArguments": [executable, "serve"],
27
+ "RunAtLoad": true,
28
+ "KeepAlive": true
29
+ ]
30
+ let data = try PropertyListSerialization.data(fromPropertyList: plist, format: .xml, options: 0)
31
+ try data.write(to: plistURL)
32
+ }
33
+
34
+ static func removePlist() throws {
35
+ try? FileManager.default.removeItem(at: plistURL)
36
+ }
37
+ }
38
+
39
+ @MainActor
40
+ struct ServiceInstall: ParsableCommand {
41
+ static var commandDescription: CommandDescription {
42
+ CommandDescription(commandName: "install", abstract: "Install user launch agent")
43
+ }
44
+
45
+ mutating func run() async throws {
46
+ let exe = CommandLine.arguments.first ?? "/usr/local/bin/swabble"
47
+ try LaunchdHelper.writePlist(executable: exe)
48
+ print("launchctl load -w \(LaunchdHelper.plistURL.path)")
49
+ }
50
+ }
51
+
52
+ @MainActor
53
+ struct ServiceUninstall: ParsableCommand {
54
+ static var commandDescription: CommandDescription {
55
+ CommandDescription(commandName: "uninstall", abstract: "Remove launch agent")
56
+ }
57
+
58
+ mutating func run() async throws {
59
+ try LaunchdHelper.removePlist()
60
+ print("launchctl bootout gui/$(id -u)/\(LaunchdHelper.label)")
61
+ }
62
+ }
63
+
64
+ @MainActor
65
+ struct ServiceStatus: ParsableCommand {
66
+ static var commandDescription: CommandDescription {
67
+ CommandDescription(commandName: "status", abstract: "Show launch agent status")
68
+ }
69
+
70
+ mutating func run() async throws {
71
+ if FileManager.default.fileExists(atPath: LaunchdHelper.plistURL.path) {
72
+ print("plist present at \(LaunchdHelper.plistURL.path)")
73
+ } else {
74
+ print("launchd plist not installed")
75
+ }
76
+ }
77
+ }
Swabble/Sources/swabble/Commands/SetupCommand.swift ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+ import Swabble
4
+
5
+ @MainActor
6
+ struct SetupCommand: ParsableCommand {
7
+ static var commandDescription: CommandDescription {
8
+ CommandDescription(commandName: "setup", abstract: "Write default config")
9
+ }
10
+
11
+ @Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
12
+
13
+ init() {}
14
+ init(parsed: ParsedValues) {
15
+ self.init()
16
+ if let cfg = parsed.options["config"]?.last { configPath = cfg }
17
+ }
18
+
19
+ mutating func run() async throws {
20
+ let cfg = SwabbleConfig()
21
+ try ConfigLoader.save(cfg, at: configURL)
22
+ print("wrote config to \(configURL?.path ?? SwabbleConfig.defaultPath.path)")
23
+ }
24
+
25
+ private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
26
+ }
Swabble/Sources/swabble/Commands/StartStopCommands.swift ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+
4
+ @MainActor
5
+ struct StartCommand: ParsableCommand {
6
+ static var commandDescription: CommandDescription {
7
+ CommandDescription(commandName: "start", abstract: "Start swabble (foreground placeholder)")
8
+ }
9
+
10
+ mutating func run() async throws {
11
+ print("start: launchd helper not implemented; run 'swabble serve' instead")
12
+ }
13
+ }
14
+
15
+ @MainActor
16
+ struct StopCommand: ParsableCommand {
17
+ static var commandDescription: CommandDescription {
18
+ CommandDescription(commandName: "stop", abstract: "Stop swabble (placeholder)")
19
+ }
20
+
21
+ mutating func run() async throws {
22
+ print("stop: launchd helper not implemented yet")
23
+ }
24
+ }
25
+
26
+ @MainActor
27
+ struct RestartCommand: ParsableCommand {
28
+ static var commandDescription: CommandDescription {
29
+ CommandDescription(commandName: "restart", abstract: "Restart swabble (placeholder)")
30
+ }
31
+
32
+ mutating func run() async throws {
33
+ print("restart: launchd helper not implemented yet")
34
+ }
35
+ }
Swabble/Sources/swabble/Commands/StatusCommand.swift ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+ import Swabble
4
+
5
+ @MainActor
6
+ struct StatusCommand: ParsableCommand {
7
+ static var commandDescription: CommandDescription {
8
+ CommandDescription(commandName: "status", abstract: "Show daemon state")
9
+ }
10
+
11
+ @Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
12
+
13
+ init() {}
14
+ init(parsed: ParsedValues) {
15
+ self.init()
16
+ if let cfg = parsed.options["config"]?.last { configPath = cfg }
17
+ }
18
+
19
+ mutating func run() async throws {
20
+ let cfg = try? ConfigLoader.load(at: configURL)
21
+ let wake = cfg?.wake.word ?? "clawd"
22
+ let wakeEnabled = cfg?.wake.enabled ?? false
23
+ let latest = await TranscriptsStore.shared.latest().suffix(3)
24
+ print("wake: \(wakeEnabled ? wake : "disabled")")
25
+ if latest.isEmpty {
26
+ print("transcripts: (none yet)")
27
+ } else {
28
+ print("last transcripts:")
29
+ latest.forEach { print("- \($0)") }
30
+ }
31
+ }
32
+
33
+ private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
34
+ }
Swabble/Sources/swabble/Commands/TailLogCommand.swift ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+ import Swabble
4
+
5
+ @MainActor
6
+ struct TailLogCommand: ParsableCommand {
7
+ static var commandDescription: CommandDescription {
8
+ CommandDescription(commandName: "tail-log", abstract: "Tail recent transcripts")
9
+ }
10
+
11
+ init() {}
12
+ init(parsed: ParsedValues) {}
13
+
14
+ mutating func run() async throws {
15
+ let latest = await TranscriptsStore.shared.latest()
16
+ for line in latest.suffix(10) {
17
+ print(line)
18
+ }
19
+ }
20
+ }
Swabble/Sources/swabble/Commands/TestHookCommand.swift ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+ import Swabble
4
+
5
+ @MainActor
6
+ struct TestHookCommand: ParsableCommand {
7
+ @Argument(help: "Text to send to hook") var text: String
8
+ @Option(name: .long("config"), help: "Path to config JSON") var configPath: String?
9
+
10
+ static var commandDescription: CommandDescription {
11
+ CommandDescription(commandName: "test-hook", abstract: "Invoke the configured hook with text")
12
+ }
13
+
14
+ init() {}
15
+
16
+ init(parsed: ParsedValues) {
17
+ self.init()
18
+ if let positional = parsed.positional.first { text = positional }
19
+ if let cfg = parsed.options["config"]?.last { configPath = cfg }
20
+ }
21
+
22
+ mutating func run() async throws {
23
+ let cfg = try ConfigLoader.load(at: configURL)
24
+ let executor = HookExecutor(config: cfg)
25
+ try await executor.run(job: HookJob(text: text, timestamp: Date()))
26
+ print("hook invoked")
27
+ }
28
+
29
+ private var configURL: URL? { configPath.map { URL(fileURLWithPath: $0) } }
30
+ }
Swabble/Sources/swabble/Commands/TranscribeCommand.swift ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import AVFoundation
2
+ import Commander
3
+ import Foundation
4
+ import Speech
5
+ import Swabble
6
+
7
+ @MainActor
8
+ struct TranscribeCommand: ParsableCommand {
9
+ @Argument(help: "Path to audio/video file") var inputFile: String = ""
10
+ @Option(name: .long("locale"), help: "Locale identifier", parsing: .singleValue) var locale: String = Locale.current
11
+ .identifier
12
+ @Flag(help: "Censor etiquette-sensitive content") var censor: Bool = false
13
+ @Option(name: .long("output"), help: "Output file path") var outputFile: String?
14
+ @Option(name: .long("format"), help: "Output format txt|srt") var format: String = "txt"
15
+ @Option(name: .long("max-length"), help: "Max sentence length for srt") var maxLength: Int = 40
16
+
17
+ static var commandDescription: CommandDescription {
18
+ CommandDescription(
19
+ commandName: "transcribe",
20
+ abstract: "Transcribe a media file locally")
21
+ }
22
+
23
+ init() {}
24
+
25
+ init(parsed: ParsedValues) {
26
+ self.init()
27
+ if let positional = parsed.positional.first { inputFile = positional }
28
+ if let loc = parsed.options["locale"]?.last { locale = loc }
29
+ if parsed.flags.contains("censor") { censor = true }
30
+ if let out = parsed.options["output"]?.last { outputFile = out }
31
+ if let fmt = parsed.options["format"]?.last { format = fmt }
32
+ if let len = parsed.options["maxLength"]?.last, let intVal = Int(len) { maxLength = intVal }
33
+ }
34
+
35
+ mutating func run() async throws {
36
+ let fileURL = URL(fileURLWithPath: inputFile)
37
+ let audioFile = try AVAudioFile(forReading: fileURL)
38
+
39
+ let outputFormat = OutputFormat(rawValue: format) ?? .txt
40
+
41
+ let transcriber = SpeechTranscriber(
42
+ locale: Locale(identifier: locale),
43
+ transcriptionOptions: censor ? [.etiquetteReplacements] : [],
44
+ reportingOptions: [],
45
+ attributeOptions: outputFormat.needsAudioTimeRange ? [.audioTimeRange] : [])
46
+ let analyzer = SpeechAnalyzer(modules: [transcriber])
47
+ try await analyzer.start(inputAudioFile: audioFile, finishAfterFile: true)
48
+
49
+ var transcript: AttributedString = ""
50
+ for try await result in transcriber.results {
51
+ transcript += result.text
52
+ }
53
+
54
+ let output = outputFormat.text(for: transcript, maxLength: maxLength)
55
+ if let path = outputFile {
56
+ try output.write(to: URL(fileURLWithPath: path), atomically: false, encoding: .utf8)
57
+ } else {
58
+ print(output)
59
+ }
60
+ }
61
+ }
Swabble/Sources/swabble/main.swift ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Commander
2
+ import Foundation
3
+
4
+ @available(macOS 26.0, *)
5
+ @MainActor
6
+ private func runCLI() async -> Int32 {
7
+ do {
8
+ let descriptors = CLIRegistry.descriptors
9
+ let program = Program(descriptors: descriptors)
10
+ let invocation = try program.resolve(argv: CommandLine.arguments)
11
+ try await dispatch(invocation: invocation)
12
+ return 0
13
+ } catch {
14
+ fputs("error: \(error)\n", stderr)
15
+ return 1
16
+ }
17
+ }
18
+
19
+ @available(macOS 26.0, *)
20
+ @MainActor
21
+ private func dispatch(invocation: CommandInvocation) async throws {
22
+ let parsed = invocation.parsedValues
23
+ let path = invocation.path
24
+ guard let first = path.first else { throw CommanderProgramError.missingCommand }
25
+
26
+ switch first {
27
+ case "swabble":
28
+ try await dispatchSwabble(parsed: parsed, path: path)
29
+ default:
30
+ throw CommanderProgramError.unknownCommand(first)
31
+ }
32
+ }
33
+
34
+ @available(macOS 26.0, *)
35
+ @MainActor
36
+ private func dispatchSwabble(parsed: ParsedValues, path: [String]) async throws {
37
+ let sub = try subcommand(path, index: 1, command: "swabble")
38
+ switch sub {
39
+ case "mic":
40
+ try await dispatchMic(parsed: parsed, path: path)
41
+ case "service":
42
+ try await dispatchService(path: path)
43
+ default:
44
+ let handlers = swabbleHandlers(parsed: parsed)
45
+ guard let handler = handlers[sub] else {
46
+ throw CommanderProgramError.unknownSubcommand(command: "swabble", name: sub)
47
+ }
48
+ try await handler()
49
+ }
50
+ }
51
+
52
+ @available(macOS 26.0, *)
53
+ @MainActor
54
+ private func swabbleHandlers(parsed: ParsedValues) -> [String: () async throws -> Void] {
55
+ [
56
+ "serve": {
57
+ var cmd = ServeCommand(parsed: parsed)
58
+ try await cmd.run()
59
+ },
60
+ "transcribe": {
61
+ var cmd = TranscribeCommand(parsed: parsed)
62
+ try await cmd.run()
63
+ },
64
+ "test-hook": {
65
+ var cmd = TestHookCommand(parsed: parsed)
66
+ try await cmd.run()
67
+ },
68
+ "doctor": {
69
+ var cmd = DoctorCommand(parsed: parsed)
70
+ try await cmd.run()
71
+ },
72
+ "setup": {
73
+ var cmd = SetupCommand(parsed: parsed)
74
+ try await cmd.run()
75
+ },
76
+ "health": {
77
+ var cmd = HealthCommand(parsed: parsed)
78
+ try await cmd.run()
79
+ },
80
+ "tail-log": {
81
+ var cmd = TailLogCommand(parsed: parsed)
82
+ try await cmd.run()
83
+ },
84
+ "start": {
85
+ var cmd = StartCommand()
86
+ try await cmd.run()
87
+ },
88
+ "stop": {
89
+ var cmd = StopCommand()
90
+ try await cmd.run()
91
+ },
92
+ "restart": {
93
+ var cmd = RestartCommand()
94
+ try await cmd.run()
95
+ },
96
+ "status": {
97
+ var cmd = StatusCommand()
98
+ try await cmd.run()
99
+ }
100
+ ]
101
+ }
102
+
103
+ @available(macOS 26.0, *)
104
+ @MainActor
105
+ private func dispatchMic(parsed: ParsedValues, path: [String]) async throws {
106
+ let micSub = try subcommand(path, index: 2, command: "mic")
107
+ switch micSub {
108
+ case "list":
109
+ var cmd = MicList(parsed: parsed)
110
+ try await cmd.run()
111
+ case "set":
112
+ var cmd = MicSet(parsed: parsed)
113
+ try await cmd.run()
114
+ default:
115
+ throw CommanderProgramError.unknownSubcommand(command: "mic", name: micSub)
116
+ }
117
+ }
118
+
119
+ @available(macOS 26.0, *)
120
+ @MainActor
121
+ private func dispatchService(path: [String]) async throws {
122
+ let svcSub = try subcommand(path, index: 2, command: "service")
123
+ switch svcSub {
124
+ case "install":
125
+ var cmd = ServiceInstall()
126
+ try await cmd.run()
127
+ case "uninstall":
128
+ var cmd = ServiceUninstall()
129
+ try await cmd.run()
130
+ case "status":
131
+ var cmd = ServiceStatus()
132
+ try await cmd.run()
133
+ default:
134
+ throw CommanderProgramError.unknownSubcommand(command: "service", name: svcSub)
135
+ }
136
+ }
137
+
138
+ private func subcommand(_ path: [String], index: Int, command: String) throws -> String {
139
+ guard path.count > index else {
140
+ throw CommanderProgramError.missingSubcommand(command: command)
141
+ }
142
+ return path[index]
143
+ }
144
+
145
+ if #available(macOS 26.0, *) {
146
+ let exitCode = await runCLI()
147
+ exit(exitCode)
148
+ } else {
149
+ fputs("error: swabble requires macOS 26 or newer\n", stderr)
150
+ exit(1)
151
+ }
Swabble/Tests/SwabbleKitTests/WakeWordGateTests.swift ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Foundation
2
+ import SwabbleKit
3
+ import Testing
4
+
5
+ @Suite struct WakeWordGateTests {
6
+ @Test func matchRequiresGapAfterTrigger() {
7
+ let transcript = "hey clawd do thing"
8
+ let segments = makeSegments(
9
+ transcript: transcript,
10
+ words: [
11
+ ("hey", 0.0, 0.1),
12
+ ("clawd", 0.2, 0.1),
13
+ ("do", 0.35, 0.1),
14
+ ("thing", 0.5, 0.1),
15
+ ])
16
+ let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
17
+ #expect(WakeWordGate.match(transcript: transcript, segments: segments, config: config) == nil)
18
+ }
19
+
20
+ @Test func matchAllowsGapAndExtractsCommand() {
21
+ let transcript = "hey clawd do thing"
22
+ let segments = makeSegments(
23
+ transcript: transcript,
24
+ words: [
25
+ ("hey", 0.0, 0.1),
26
+ ("clawd", 0.2, 0.1),
27
+ ("do", 0.9, 0.1),
28
+ ("thing", 1.1, 0.1),
29
+ ])
30
+ let config = WakeWordGateConfig(triggers: ["clawd"], minPostTriggerGap: 0.3)
31
+ let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
32
+ #expect(match?.command == "do thing")
33
+ }
34
+
35
+ @Test func matchHandlesMultiWordTriggers() {
36
+ let transcript = "hey clawd do it"
37
+ let segments = makeSegments(
38
+ transcript: transcript,
39
+ words: [
40
+ ("hey", 0.0, 0.1),
41
+ ("clawd", 0.2, 0.1),
42
+ ("do", 0.8, 0.1),
43
+ ("it", 1.0, 0.1),
44
+ ])
45
+ let config = WakeWordGateConfig(triggers: ["hey clawd"], minPostTriggerGap: 0.3)
46
+ let match = WakeWordGate.match(transcript: transcript, segments: segments, config: config)
47
+ #expect(match?.command == "do it")
48
+ }
49
+ }
50
+
51
+ private func makeSegments(
52
+ transcript: String,
53
+ words: [(String, TimeInterval, TimeInterval)])
54
+ -> [WakeWordSegment] {
55
+ var searchStart = transcript.startIndex
56
+ var output: [WakeWordSegment] = []
57
+ for (word, start, duration) in words {
58
+ let range = transcript.range(of: word, range: searchStart..<transcript.endIndex)
59
+ output.append(WakeWordSegment(text: word, start: start, duration: duration, range: range))
60
+ if let range { searchStart = range.upperBound }
61
+ }
62
+ return output
63
+ }
Swabble/Tests/swabbleTests/ConfigTests.swift ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import Foundation
2
+ import Testing
3
+ @testable import Swabble
4
+
5
+ @Test
6
+ func configRoundTrip() throws {
7
+ var cfg = SwabbleConfig()
8
+ cfg.wake.word = "robot"
9
+ let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".json")
10
+ defer { try? FileManager.default.removeItem(at: url) }
11
+
12
+ try ConfigLoader.save(cfg, at: url)
13
+ let loaded = try ConfigLoader.load(at: url)
14
+ #expect(loaded.wake.word == "robot")
15
+ #expect(loaded.hook.prefix.contains("Voice swabble"))
16
+ }
17
+
18
+ @Test
19
+ func configMissingThrows() {
20
+ #expect(throws: ConfigError.missingConfig) {
21
+ _ = try ConfigLoader.load(at: FileManager.default.temporaryDirectory.appendingPathComponent("nope.json"))
22
+ }
23
+ }
Swabble/docs/spec.md ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # swabble — macOS 26 speech hook daemon (Swift 6.2)
2
+
3
+ Goal: brabble-style always-on voice hook for macOS 26 using Apple Speech.framework (SpeechAnalyzer + SpeechTranscriber) instead of whisper.cpp. Local-only, wake word gated, dispatches a shell hook with the transcript. Shared wake-gate utilities live in `SwabbleKit` for reuse by other apps (iOS/macOS).
4
+
5
+ ## Requirements
6
+ - macOS 26+, Swift 6.2, Speech.framework with on-device assets.
7
+ - Local only; no network calls during transcription.
8
+ - Wake word gating (default "clawd" plus aliases) with bypass flag `--no-wake`.
9
+ - `SwabbleKit` target (multi-platform) providing wake-word gating helpers that can use speech segment timing to require a post-trigger gap.
10
+ - Hook execution with cooldown, min_chars, timeout, prefix, env vars.
11
+ - Simple config at `~/.config/swabble/config.json` (JSON, Codable) — no TOML.
12
+ - CLI implemented with Commander (SwiftPM package `steipete/Commander`); core types are available via the SwiftPM library product `Swabble` for embedding.
13
+ - Foreground `serve`; later launchd helper for start/stop/restart.
14
+ - File transcription command emitting txt or srt.
15
+ - Basic status/health surfaces and mic selection stubs.
16
+
17
+ ## Architecture
18
+ - **CLI layer (Commander)**: Root command `swabble` with subcommands `serve`, `transcribe`, `test-hook`, `mic list|set`, `doctor`, `health`, `tail-log`. Runtime flags from Commander (`-v/--verbose`, `--json-output`, `--log-level`). Custom `--config` path applies everywhere.
19
+ - **Config**: `SwabbleConfig` Codable. Fields: audio device name/index, wake (enabled/word/aliases/sensitivity placeholder), hook (command/args/prefix/cooldown/min_chars/timeout/env), logging (level, format), transcripts (enabled, max kept), speech (locale, enableEtiquetteReplacements flag). Stored JSON; default written by `setup`.
20
+ - **Audio + Speech pipeline**: `SpeechPipeline` wraps `AVAudioEngine` input → `SpeechAnalyzer` with `SpeechTranscriber` module. Emits partial/final transcripts via async stream. Requests `.audioTimeRange` when transcripts enabled. Handles Speech permission and asset download prompts ahead of capture.
21
+ - **Wake gate**: CLI currently uses text-only keyword match; shared `SwabbleKit` gate can enforce a minimum pause between the wake word and the next token when speech segments are available. `--no-wake` disables gating.
22
+ - **Hook executor**: async `HookExecutor` spawns `Process` with configured args, prefix substitution `${hostname}`. Enforces cooldown + timeout; injects env `SWABBLE_TEXT`, `SWABBLE_PREFIX` plus user env map.
23
+ - **Transcripts store**: in-memory ring buffer; optional persisted JSON lines under `~/Library/Application Support/swabble/transcripts.log`.
24
+ - **Logging**: simple structured logger to stderr; respects log level.
25
+
26
+ ## Out of scope (initial cut)
27
+ - Model management (Speech handles assets).
28
+ - Launchd helper (planned follow-up).
29
+ - Advanced wake-word detector (segment-aware gate now lives in `SwabbleKit`; CLI still text-only until segment timing is plumbed through).
30
+
31
+ ## Open decisions
32
+ - Whether to expose a UNIX control socket for `status`/`health` (currently planned as stdin/out direct calls).
33
+ - Hook redaction (PII) parity with brabble — placeholder boolean, no implementation yet.
Swabble/scripts/format.sh ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ ROOT="$(cd "$(dirname "$0")/.." && pwd)"
4
+ CONFIG="${ROOT}/.swiftformat"
5
+ swiftformat --config "$CONFIG" "$ROOT/Sources"
Swabble/scripts/lint.sh ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ ROOT="$(cd "$(dirname "$0")/.." && pwd)"
4
+ CONFIG="${ROOT}/.swiftlint.yml"
5
+ if ! command -v swiftlint >/dev/null; then
6
+ echo "swiftlint not installed" >&2
7
+ exit 1
8
+ fi
9
+ swiftlint --config "$CONFIG"
apps/android/.gitignore ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ .gradle/
2
+ **/build/
3
+ local.properties
4
+ .idea/
5
+ **/*.iml
apps/android/README.md ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## OpenClaw Node (Android) (internal)
2
+
3
+ Modern Android node app: connects to the **Gateway WebSocket** (`_openclaw-gw._tcp`) and exposes **Canvas + Chat + Camera**.
4
+
5
+ Notes:
6
+ - The node keeps the connection alive via a **foreground service** (persistent notification with a Disconnect action).
7
+ - Chat always uses the shared session key **`main`** (same session across iOS/macOS/WebChat/Android).
8
+ - Supports modern Android only (`minSdk 31`, Kotlin + Jetpack Compose).
9
+
10
+ ## Open in Android Studio
11
+ - Open the folder `apps/android`.
12
+
13
+ ## Build / Run
14
+
15
+ ```bash
16
+ cd apps/android
17
+ ./gradlew :app:assembleDebug
18
+ ./gradlew :app:installDebug
19
+ ./gradlew :app:testDebugUnitTest
20
+ ```
21
+
22
+ `gradlew` auto-detects the Android SDK at `~/Library/Android/sdk` (macOS default) if `ANDROID_SDK_ROOT` / `ANDROID_HOME` are unset.
23
+
24
+ ## Connect / Pair
25
+
26
+ 1) Start the gateway (on your “master” machine):
27
+ ```bash
28
+ pnpm openclaw gateway --port 18789 --verbose
29
+ ```
30
+
31
+ 2) In the Android app:
32
+ - Open **Settings**
33
+ - Either select a discovered gateway under **Discovered Gateways**, or use **Advanced → Manual Gateway** (host + port).
34
+
35
+ 3) Approve pairing (on the gateway machine):
36
+ ```bash
37
+ openclaw nodes pending
38
+ openclaw nodes approve <requestId>
39
+ ```
40
+
41
+ More details: `docs/platforms/android.md`.
42
+
43
+ ## Permissions
44
+
45
+ - Discovery:
46
+ - Android 13+ (`API 33+`): `NEARBY_WIFI_DEVICES`
47
+ - Android 12 and below: `ACCESS_FINE_LOCATION` (required for NSD scanning)
48
+ - Foreground service notification (Android 13+): `POST_NOTIFICATIONS`
49
+ - Camera:
50
+ - `CAMERA` for `camera.snap` and `camera.clip`
51
+ - `RECORD_AUDIO` for `camera.clip` when `includeAudio=true`
apps/android/app/build.gradle.kts ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import com.android.build.api.variant.impl.VariantOutputImpl
2
+
3
+ plugins {
4
+ id("com.android.application")
5
+ id("org.jetbrains.kotlin.android")
6
+ id("org.jetbrains.kotlin.plugin.compose")
7
+ id("org.jetbrains.kotlin.plugin.serialization")
8
+ }
9
+
10
+ android {
11
+ namespace = "ai.openclaw.android"
12
+ compileSdk = 36
13
+
14
+ sourceSets {
15
+ getByName("main") {
16
+ assets.srcDir(file("../../shared/OpenClawKit/Sources/OpenClawKit/Resources"))
17
+ }
18
+ }
19
+
20
+ defaultConfig {
21
+ applicationId = "ai.openclaw.android"
22
+ minSdk = 31
23
+ targetSdk = 36
24
+ versionCode = 202601290
25
+ versionName = "2026.1.30"
26
+ }
27
+
28
+ buildTypes {
29
+ release {
30
+ isMinifyEnabled = false
31
+ }
32
+ }
33
+
34
+ buildFeatures {
35
+ compose = true
36
+ buildConfig = true
37
+ }
38
+
39
+ compileOptions {
40
+ sourceCompatibility = JavaVersion.VERSION_17
41
+ targetCompatibility = JavaVersion.VERSION_17
42
+ }
43
+
44
+ packaging {
45
+ resources {
46
+ excludes += "/META-INF/{AL2.0,LGPL2.1}"
47
+ }
48
+ }
49
+
50
+ lint {
51
+ disable += setOf("IconLauncherShape")
52
+ warningsAsErrors = true
53
+ }
54
+
55
+ testOptions {
56
+ unitTests.isIncludeAndroidResources = true
57
+ }
58
+ }
59
+
60
+ androidComponents {
61
+ onVariants { variant ->
62
+ variant.outputs
63
+ .filterIsInstance<VariantOutputImpl>()
64
+ .forEach { output ->
65
+ val versionName = output.versionName.orNull ?: "0"
66
+ val buildType = variant.buildType
67
+
68
+ val outputFileName = "openclaw-${versionName}-${buildType}.apk"
69
+ output.outputFileName = outputFileName
70
+ }
71
+ }
72
+ }
73
+ kotlin {
74
+ compilerOptions {
75
+ jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17)
76
+ allWarningsAsErrors.set(true)
77
+ }
78
+ }
79
+
80
+ dependencies {
81
+ val composeBom = platform("androidx.compose:compose-bom:2025.12.00")
82
+ implementation(composeBom)
83
+ androidTestImplementation(composeBom)
84
+
85
+ implementation("androidx.core:core-ktx:1.17.0")
86
+ implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.10.0")
87
+ implementation("androidx.activity:activity-compose:1.12.2")
88
+ implementation("androidx.webkit:webkit:1.15.0")
89
+
90
+ implementation("androidx.compose.ui:ui")
91
+ implementation("androidx.compose.ui:ui-tooling-preview")
92
+ implementation("androidx.compose.material3:material3")
93
+ implementation("androidx.compose.material:material-icons-extended")
94
+ implementation("androidx.navigation:navigation-compose:2.9.6")
95
+
96
+ debugImplementation("androidx.compose.ui:ui-tooling")
97
+
98
+ // Material Components (XML theme + resources)
99
+ implementation("com.google.android.material:material:1.13.0")
100
+
101
+ implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.10.2")
102
+ implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0")
103
+
104
+ implementation("androidx.security:security-crypto:1.1.0")
105
+ implementation("androidx.exifinterface:exifinterface:1.4.2")
106
+ implementation("com.squareup.okhttp3:okhttp:5.3.2")
107
+
108
+ // CameraX (for node.invoke camera.* parity)
109
+ implementation("androidx.camera:camera-core:1.5.2")
110
+ implementation("androidx.camera:camera-camera2:1.5.2")
111
+ implementation("androidx.camera:camera-lifecycle:1.5.2")
112
+ implementation("androidx.camera:camera-video:1.5.2")
113
+ implementation("androidx.camera:camera-view:1.5.2")
114
+
115
+ // Unicast DNS-SD (Wide-Area Bonjour) for tailnet discovery domains.
116
+ implementation("dnsjava:dnsjava:3.6.4")
117
+
118
+ testImplementation("junit:junit:4.13.2")
119
+ testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2")
120
+ testImplementation("io.kotest:kotest-runner-junit5-jvm:6.0.7")
121
+ testImplementation("io.kotest:kotest-assertions-core-jvm:6.0.7")
122
+ testImplementation("org.robolectric:robolectric:4.16")
123
+ testRuntimeOnly("org.junit.vintage:junit-vintage-engine:6.0.2")
124
+ }
125
+
126
+ tasks.withType<Test>().configureEach {
127
+ useJUnitPlatform()
128
+ }
apps/android/app/src/main/AndroidManifest.xml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <manifest xmlns:android="http://schemas.android.com/apk/res/android">
2
+ <uses-permission android:name="android.permission.INTERNET" />
3
+ <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
4
+ <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
5
+ <uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
6
+ <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
7
+ <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
8
+ <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
9
+ <uses-permission
10
+ android:name="android.permission.NEARBY_WIFI_DEVICES"
11
+ android:usesPermissionFlags="neverForLocation" />
12
+ <uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
13
+ <uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
14
+ <uses-permission android:name="android.permission.ACCESS_BACKGROUND_LOCATION" />
15
+ <uses-permission android:name="android.permission.CAMERA" />
16
+ <uses-permission android:name="android.permission.RECORD_AUDIO" />
17
+ <uses-permission android:name="android.permission.SEND_SMS" />
18
+ <uses-feature
19
+ android:name="android.hardware.camera"
20
+ android:required="false" />
21
+ <uses-feature
22
+ android:name="android.hardware.telephony"
23
+ android:required="false" />
24
+
25
+ <application
26
+ android:name=".NodeApp"
27
+ android:allowBackup="true"
28
+ android:dataExtractionRules="@xml/data_extraction_rules"
29
+ android:fullBackupContent="@xml/backup_rules"
30
+ android:icon="@mipmap/ic_launcher"
31
+ android:roundIcon="@mipmap/ic_launcher_round"
32
+ android:label="@string/app_name"
33
+ android:supportsRtl="true"
34
+ android:networkSecurityConfig="@xml/network_security_config"
35
+ android:theme="@style/Theme.OpenClawNode">
36
+ <service
37
+ android:name=".NodeForegroundService"
38
+ android:exported="false"
39
+ android:foregroundServiceType="dataSync|microphone|mediaProjection" />
40
+ <activity
41
+ android:name=".MainActivity"
42
+ android:exported="true">
43
+ <intent-filter>
44
+ <action android:name="android.intent.action.MAIN" />
45
+ <category android:name="android.intent.category.LAUNCHER" />
46
+ </intent-filter>
47
+ </activity>
48
+ </application>
49
+ </manifest>
apps/android/app/src/main/java/ai/openclaw/android/CameraHudState.kt ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ enum class CameraHudKind {
4
+ Photo,
5
+ Recording,
6
+ Success,
7
+ Error,
8
+ }
9
+
10
+ data class CameraHudState(
11
+ val token: Long,
12
+ val kind: CameraHudKind,
13
+ val message: String,
14
+ )
apps/android/app/src/main/java/ai/openclaw/android/DeviceNames.kt ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ import android.content.Context
4
+ import android.os.Build
5
+ import android.provider.Settings
6
+
7
+ object DeviceNames {
8
+ fun bestDefaultNodeName(context: Context): String {
9
+ val deviceName =
10
+ runCatching {
11
+ Settings.Global.getString(context.contentResolver, "device_name")
12
+ }
13
+ .getOrNull()
14
+ ?.trim()
15
+ .orEmpty()
16
+
17
+ if (deviceName.isNotEmpty()) return deviceName
18
+
19
+ val model =
20
+ listOfNotNull(Build.MANUFACTURER?.takeIf { it.isNotBlank() }, Build.MODEL?.takeIf { it.isNotBlank() })
21
+ .joinToString(" ")
22
+ .trim()
23
+
24
+ return model.ifEmpty { "Android Node" }
25
+ }
26
+ }
apps/android/app/src/main/java/ai/openclaw/android/LocationMode.kt ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ enum class LocationMode(val rawValue: String) {
4
+ Off("off"),
5
+ WhileUsing("whileUsing"),
6
+ Always("always"),
7
+ ;
8
+
9
+ companion object {
10
+ fun fromRawValue(raw: String?): LocationMode {
11
+ val normalized = raw?.trim()?.lowercase()
12
+ return entries.firstOrNull { it.rawValue.lowercase() == normalized } ?: Off
13
+ }
14
+ }
15
+ }
apps/android/app/src/main/java/ai/openclaw/android/MainActivity.kt ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ import android.Manifest
4
+ import android.content.pm.ApplicationInfo
5
+ import android.os.Bundle
6
+ import android.os.Build
7
+ import android.view.WindowManager
8
+ import android.webkit.WebView
9
+ import androidx.activity.ComponentActivity
10
+ import androidx.activity.compose.setContent
11
+ import androidx.activity.viewModels
12
+ import androidx.compose.material3.Surface
13
+ import androidx.compose.ui.Modifier
14
+ import androidx.core.content.ContextCompat
15
+ import androidx.core.view.WindowCompat
16
+ import androidx.core.view.WindowInsetsCompat
17
+ import androidx.core.view.WindowInsetsControllerCompat
18
+ import androidx.lifecycle.Lifecycle
19
+ import androidx.lifecycle.lifecycleScope
20
+ import androidx.lifecycle.repeatOnLifecycle
21
+ import ai.openclaw.android.ui.RootScreen
22
+ import ai.openclaw.android.ui.OpenClawTheme
23
+ import kotlinx.coroutines.launch
24
+
25
+ class MainActivity : ComponentActivity() {
26
+ private val viewModel: MainViewModel by viewModels()
27
+ private lateinit var permissionRequester: PermissionRequester
28
+ private lateinit var screenCaptureRequester: ScreenCaptureRequester
29
+
30
+ override fun onCreate(savedInstanceState: Bundle?) {
31
+ super.onCreate(savedInstanceState)
32
+ val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0
33
+ WebView.setWebContentsDebuggingEnabled(isDebuggable)
34
+ applyImmersiveMode()
35
+ requestDiscoveryPermissionsIfNeeded()
36
+ requestNotificationPermissionIfNeeded()
37
+ NodeForegroundService.start(this)
38
+ permissionRequester = PermissionRequester(this)
39
+ screenCaptureRequester = ScreenCaptureRequester(this)
40
+ viewModel.camera.attachLifecycleOwner(this)
41
+ viewModel.camera.attachPermissionRequester(permissionRequester)
42
+ viewModel.sms.attachPermissionRequester(permissionRequester)
43
+ viewModel.screenRecorder.attachScreenCaptureRequester(screenCaptureRequester)
44
+ viewModel.screenRecorder.attachPermissionRequester(permissionRequester)
45
+
46
+ lifecycleScope.launch {
47
+ repeatOnLifecycle(Lifecycle.State.STARTED) {
48
+ viewModel.preventSleep.collect { enabled ->
49
+ if (enabled) {
50
+ window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
51
+ } else {
52
+ window.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
53
+ }
54
+ }
55
+ }
56
+ }
57
+
58
+ setContent {
59
+ OpenClawTheme {
60
+ Surface(modifier = Modifier) {
61
+ RootScreen(viewModel = viewModel)
62
+ }
63
+ }
64
+ }
65
+ }
66
+
67
+ override fun onResume() {
68
+ super.onResume()
69
+ applyImmersiveMode()
70
+ }
71
+
72
+ override fun onWindowFocusChanged(hasFocus: Boolean) {
73
+ super.onWindowFocusChanged(hasFocus)
74
+ if (hasFocus) {
75
+ applyImmersiveMode()
76
+ }
77
+ }
78
+
79
+ override fun onStart() {
80
+ super.onStart()
81
+ viewModel.setForeground(true)
82
+ }
83
+
84
+ override fun onStop() {
85
+ viewModel.setForeground(false)
86
+ super.onStop()
87
+ }
88
+
89
+ private fun applyImmersiveMode() {
90
+ WindowCompat.setDecorFitsSystemWindows(window, false)
91
+ val controller = WindowInsetsControllerCompat(window, window.decorView)
92
+ controller.systemBarsBehavior =
93
+ WindowInsetsControllerCompat.BEHAVIOR_SHOW_TRANSIENT_BARS_BY_SWIPE
94
+ controller.hide(WindowInsetsCompat.Type.systemBars())
95
+ }
96
+
97
+ private fun requestDiscoveryPermissionsIfNeeded() {
98
+ if (Build.VERSION.SDK_INT >= 33) {
99
+ val ok =
100
+ ContextCompat.checkSelfPermission(
101
+ this,
102
+ Manifest.permission.NEARBY_WIFI_DEVICES,
103
+ ) == android.content.pm.PackageManager.PERMISSION_GRANTED
104
+ if (!ok) {
105
+ requestPermissions(arrayOf(Manifest.permission.NEARBY_WIFI_DEVICES), 100)
106
+ }
107
+ } else {
108
+ val ok =
109
+ ContextCompat.checkSelfPermission(
110
+ this,
111
+ Manifest.permission.ACCESS_FINE_LOCATION,
112
+ ) == android.content.pm.PackageManager.PERMISSION_GRANTED
113
+ if (!ok) {
114
+ requestPermissions(arrayOf(Manifest.permission.ACCESS_FINE_LOCATION), 101)
115
+ }
116
+ }
117
+ }
118
+
119
+ private fun requestNotificationPermissionIfNeeded() {
120
+ if (Build.VERSION.SDK_INT < 33) return
121
+ val ok =
122
+ ContextCompat.checkSelfPermission(
123
+ this,
124
+ Manifest.permission.POST_NOTIFICATIONS,
125
+ ) == android.content.pm.PackageManager.PERMISSION_GRANTED
126
+ if (!ok) {
127
+ requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), 102)
128
+ }
129
+ }
130
+ }
apps/android/app/src/main/java/ai/openclaw/android/MainViewModel.kt ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ import android.app.Application
4
+ import androidx.lifecycle.AndroidViewModel
5
+ import ai.openclaw.android.gateway.GatewayEndpoint
6
+ import ai.openclaw.android.chat.OutgoingAttachment
7
+ import ai.openclaw.android.node.CameraCaptureManager
8
+ import ai.openclaw.android.node.CanvasController
9
+ import ai.openclaw.android.node.ScreenRecordManager
10
+ import ai.openclaw.android.node.SmsManager
11
+ import kotlinx.coroutines.flow.StateFlow
12
+
13
+ class MainViewModel(app: Application) : AndroidViewModel(app) {
14
+ private val runtime: NodeRuntime = (app as NodeApp).runtime
15
+
16
+ val canvas: CanvasController = runtime.canvas
17
+ val camera: CameraCaptureManager = runtime.camera
18
+ val screenRecorder: ScreenRecordManager = runtime.screenRecorder
19
+ val sms: SmsManager = runtime.sms
20
+
21
+ val gateways: StateFlow<List<GatewayEndpoint>> = runtime.gateways
22
+ val discoveryStatusText: StateFlow<String> = runtime.discoveryStatusText
23
+
24
+ val isConnected: StateFlow<Boolean> = runtime.isConnected
25
+ val statusText: StateFlow<String> = runtime.statusText
26
+ val serverName: StateFlow<String?> = runtime.serverName
27
+ val remoteAddress: StateFlow<String?> = runtime.remoteAddress
28
+ val isForeground: StateFlow<Boolean> = runtime.isForeground
29
+ val seamColorArgb: StateFlow<Long> = runtime.seamColorArgb
30
+ val mainSessionKey: StateFlow<String> = runtime.mainSessionKey
31
+
32
+ val cameraHud: StateFlow<CameraHudState?> = runtime.cameraHud
33
+ val cameraFlashToken: StateFlow<Long> = runtime.cameraFlashToken
34
+ val screenRecordActive: StateFlow<Boolean> = runtime.screenRecordActive
35
+
36
+ val instanceId: StateFlow<String> = runtime.instanceId
37
+ val displayName: StateFlow<String> = runtime.displayName
38
+ val cameraEnabled: StateFlow<Boolean> = runtime.cameraEnabled
39
+ val locationMode: StateFlow<LocationMode> = runtime.locationMode
40
+ val locationPreciseEnabled: StateFlow<Boolean> = runtime.locationPreciseEnabled
41
+ val preventSleep: StateFlow<Boolean> = runtime.preventSleep
42
+ val wakeWords: StateFlow<List<String>> = runtime.wakeWords
43
+ val voiceWakeMode: StateFlow<VoiceWakeMode> = runtime.voiceWakeMode
44
+ val voiceWakeStatusText: StateFlow<String> = runtime.voiceWakeStatusText
45
+ val voiceWakeIsListening: StateFlow<Boolean> = runtime.voiceWakeIsListening
46
+ val talkEnabled: StateFlow<Boolean> = runtime.talkEnabled
47
+ val talkStatusText: StateFlow<String> = runtime.talkStatusText
48
+ val talkIsListening: StateFlow<Boolean> = runtime.talkIsListening
49
+ val talkIsSpeaking: StateFlow<Boolean> = runtime.talkIsSpeaking
50
+ val manualEnabled: StateFlow<Boolean> = runtime.manualEnabled
51
+ val manualHost: StateFlow<String> = runtime.manualHost
52
+ val manualPort: StateFlow<Int> = runtime.manualPort
53
+ val manualTls: StateFlow<Boolean> = runtime.manualTls
54
+ val canvasDebugStatusEnabled: StateFlow<Boolean> = runtime.canvasDebugStatusEnabled
55
+
56
+ val chatSessionKey: StateFlow<String> = runtime.chatSessionKey
57
+ val chatSessionId: StateFlow<String?> = runtime.chatSessionId
58
+ val chatMessages = runtime.chatMessages
59
+ val chatError: StateFlow<String?> = runtime.chatError
60
+ val chatHealthOk: StateFlow<Boolean> = runtime.chatHealthOk
61
+ val chatThinkingLevel: StateFlow<String> = runtime.chatThinkingLevel
62
+ val chatStreamingAssistantText: StateFlow<String?> = runtime.chatStreamingAssistantText
63
+ val chatPendingToolCalls = runtime.chatPendingToolCalls
64
+ val chatSessions = runtime.chatSessions
65
+ val pendingRunCount: StateFlow<Int> = runtime.pendingRunCount
66
+
67
+ fun setForeground(value: Boolean) {
68
+ runtime.setForeground(value)
69
+ }
70
+
71
+ fun setDisplayName(value: String) {
72
+ runtime.setDisplayName(value)
73
+ }
74
+
75
+ fun setCameraEnabled(value: Boolean) {
76
+ runtime.setCameraEnabled(value)
77
+ }
78
+
79
+ fun setLocationMode(mode: LocationMode) {
80
+ runtime.setLocationMode(mode)
81
+ }
82
+
83
+ fun setLocationPreciseEnabled(value: Boolean) {
84
+ runtime.setLocationPreciseEnabled(value)
85
+ }
86
+
87
+ fun setPreventSleep(value: Boolean) {
88
+ runtime.setPreventSleep(value)
89
+ }
90
+
91
+ fun setManualEnabled(value: Boolean) {
92
+ runtime.setManualEnabled(value)
93
+ }
94
+
95
+ fun setManualHost(value: String) {
96
+ runtime.setManualHost(value)
97
+ }
98
+
99
+ fun setManualPort(value: Int) {
100
+ runtime.setManualPort(value)
101
+ }
102
+
103
+ fun setManualTls(value: Boolean) {
104
+ runtime.setManualTls(value)
105
+ }
106
+
107
+ fun setCanvasDebugStatusEnabled(value: Boolean) {
108
+ runtime.setCanvasDebugStatusEnabled(value)
109
+ }
110
+
111
+ fun setWakeWords(words: List<String>) {
112
+ runtime.setWakeWords(words)
113
+ }
114
+
115
+ fun resetWakeWordsDefaults() {
116
+ runtime.resetWakeWordsDefaults()
117
+ }
118
+
119
+ fun setVoiceWakeMode(mode: VoiceWakeMode) {
120
+ runtime.setVoiceWakeMode(mode)
121
+ }
122
+
123
+ fun setTalkEnabled(enabled: Boolean) {
124
+ runtime.setTalkEnabled(enabled)
125
+ }
126
+
127
+ fun refreshGatewayConnection() {
128
+ runtime.refreshGatewayConnection()
129
+ }
130
+
131
+ fun connect(endpoint: GatewayEndpoint) {
132
+ runtime.connect(endpoint)
133
+ }
134
+
135
+ fun connectManual() {
136
+ runtime.connectManual()
137
+ }
138
+
139
+ fun disconnect() {
140
+ runtime.disconnect()
141
+ }
142
+
143
+ fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
144
+ runtime.handleCanvasA2UIActionFromWebView(payloadJson)
145
+ }
146
+
147
+ fun loadChat(sessionKey: String) {
148
+ runtime.loadChat(sessionKey)
149
+ }
150
+
151
+ fun refreshChat() {
152
+ runtime.refreshChat()
153
+ }
154
+
155
+ fun refreshChatSessions(limit: Int? = null) {
156
+ runtime.refreshChatSessions(limit = limit)
157
+ }
158
+
159
+ fun setChatThinkingLevel(level: String) {
160
+ runtime.setChatThinkingLevel(level)
161
+ }
162
+
163
+ fun switchChatSession(sessionKey: String) {
164
+ runtime.switchChatSession(sessionKey)
165
+ }
166
+
167
+ fun abortChat() {
168
+ runtime.abortChat()
169
+ }
170
+
171
+ fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
172
+ runtime.sendChat(message = message, thinking = thinking, attachments = attachments)
173
+ }
174
+ }
apps/android/app/src/main/java/ai/openclaw/android/NodeApp.kt ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ import android.app.Application
4
+ import android.os.StrictMode
5
+
6
+ class NodeApp : Application() {
7
+ val runtime: NodeRuntime by lazy { NodeRuntime(this) }
8
+
9
+ override fun onCreate() {
10
+ super.onCreate()
11
+ if (BuildConfig.DEBUG) {
12
+ StrictMode.setThreadPolicy(
13
+ StrictMode.ThreadPolicy.Builder()
14
+ .detectAll()
15
+ .penaltyLog()
16
+ .build(),
17
+ )
18
+ StrictMode.setVmPolicy(
19
+ StrictMode.VmPolicy.Builder()
20
+ .detectAll()
21
+ .penaltyLog()
22
+ .build(),
23
+ )
24
+ }
25
+ }
26
+ }
apps/android/app/src/main/java/ai/openclaw/android/NodeForegroundService.kt ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ import android.app.Notification
4
+ import android.app.NotificationChannel
5
+ import android.app.NotificationManager
6
+ import android.app.Service
7
+ import android.app.PendingIntent
8
+ import android.Manifest
9
+ import android.content.Context
10
+ import android.content.Intent
11
+ import android.content.pm.PackageManager
12
+ import android.content.pm.ServiceInfo
13
+ import androidx.core.app.NotificationCompat
14
+ import androidx.core.content.ContextCompat
15
+ import kotlinx.coroutines.CoroutineScope
16
+ import kotlinx.coroutines.Dispatchers
17
+ import kotlinx.coroutines.Job
18
+ import kotlinx.coroutines.SupervisorJob
19
+ import kotlinx.coroutines.cancel
20
+ import kotlinx.coroutines.flow.combine
21
+ import kotlinx.coroutines.launch
22
+
23
+ class NodeForegroundService : Service() {
24
+ private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
25
+ private var notificationJob: Job? = null
26
+ private var lastRequiresMic = false
27
+ private var didStartForeground = false
28
+
29
+ override fun onCreate() {
30
+ super.onCreate()
31
+ ensureChannel()
32
+ val initial = buildNotification(title = "OpenClaw Node", text = "Starting…")
33
+ startForegroundWithTypes(notification = initial, requiresMic = false)
34
+
35
+ val runtime = (application as NodeApp).runtime
36
+ notificationJob =
37
+ scope.launch {
38
+ combine(
39
+ runtime.statusText,
40
+ runtime.serverName,
41
+ runtime.isConnected,
42
+ runtime.voiceWakeMode,
43
+ runtime.voiceWakeIsListening,
44
+ ) { status, server, connected, voiceMode, voiceListening ->
45
+ Quint(status, server, connected, voiceMode, voiceListening)
46
+ }.collect { (status, server, connected, voiceMode, voiceListening) ->
47
+ val title = if (connected) "OpenClaw Node · Connected" else "OpenClaw Node"
48
+ val voiceSuffix =
49
+ if (voiceMode == VoiceWakeMode.Always) {
50
+ if (voiceListening) " · Voice Wake: Listening" else " · Voice Wake: Paused"
51
+ } else {
52
+ ""
53
+ }
54
+ val text = (server?.let { "$status · $it" } ?: status) + voiceSuffix
55
+
56
+ val requiresMic =
57
+ voiceMode == VoiceWakeMode.Always && hasRecordAudioPermission()
58
+ startForegroundWithTypes(
59
+ notification = buildNotification(title = title, text = text),
60
+ requiresMic = requiresMic,
61
+ )
62
+ }
63
+ }
64
+ }
65
+
66
+ override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
67
+ when (intent?.action) {
68
+ ACTION_STOP -> {
69
+ (application as NodeApp).runtime.disconnect()
70
+ stopSelf()
71
+ return START_NOT_STICKY
72
+ }
73
+ }
74
+ // Keep running; connection is managed by NodeRuntime (auto-reconnect + manual).
75
+ return START_STICKY
76
+ }
77
+
78
+ override fun onDestroy() {
79
+ notificationJob?.cancel()
80
+ scope.cancel()
81
+ super.onDestroy()
82
+ }
83
+
84
+ override fun onBind(intent: Intent?) = null
85
+
86
+ private fun ensureChannel() {
87
+ val mgr = getSystemService(NotificationManager::class.java)
88
+ val channel =
89
+ NotificationChannel(
90
+ CHANNEL_ID,
91
+ "Connection",
92
+ NotificationManager.IMPORTANCE_LOW,
93
+ ).apply {
94
+ description = "OpenClaw node connection status"
95
+ setShowBadge(false)
96
+ }
97
+ mgr.createNotificationChannel(channel)
98
+ }
99
+
100
+ private fun buildNotification(title: String, text: String): Notification {
101
+ val launchIntent = Intent(this, MainActivity::class.java).apply {
102
+ flags = Intent.FLAG_ACTIVITY_SINGLE_TOP or Intent.FLAG_ACTIVITY_CLEAR_TOP
103
+ }
104
+ val launchPending =
105
+ PendingIntent.getActivity(
106
+ this,
107
+ 1,
108
+ launchIntent,
109
+ PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
110
+ )
111
+
112
+ val stopIntent = Intent(this, NodeForegroundService::class.java).setAction(ACTION_STOP)
113
+ val stopPending =
114
+ PendingIntent.getService(
115
+ this,
116
+ 2,
117
+ stopIntent,
118
+ PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
119
+ )
120
+
121
+ return NotificationCompat.Builder(this, CHANNEL_ID)
122
+ .setSmallIcon(R.mipmap.ic_launcher)
123
+ .setContentTitle(title)
124
+ .setContentText(text)
125
+ .setContentIntent(launchPending)
126
+ .setOngoing(true)
127
+ .setOnlyAlertOnce(true)
128
+ .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE)
129
+ .addAction(0, "Disconnect", stopPending)
130
+ .build()
131
+ }
132
+
133
+ private fun updateNotification(notification: Notification) {
134
+ val mgr = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager
135
+ mgr.notify(NOTIFICATION_ID, notification)
136
+ }
137
+
138
+ private fun startForegroundWithTypes(notification: Notification, requiresMic: Boolean) {
139
+ if (didStartForeground && requiresMic == lastRequiresMic) {
140
+ updateNotification(notification)
141
+ return
142
+ }
143
+
144
+ lastRequiresMic = requiresMic
145
+ val types =
146
+ if (requiresMic) {
147
+ ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
148
+ } else {
149
+ ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC
150
+ }
151
+ startForeground(NOTIFICATION_ID, notification, types)
152
+ didStartForeground = true
153
+ }
154
+
155
+ private fun hasRecordAudioPermission(): Boolean {
156
+ return (
157
+ ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) ==
158
+ PackageManager.PERMISSION_GRANTED
159
+ )
160
+ }
161
+
162
+ companion object {
163
+ private const val CHANNEL_ID = "connection"
164
+ private const val NOTIFICATION_ID = 1
165
+
166
+ private const val ACTION_STOP = "ai.openclaw.android.action.STOP"
167
+
168
+ fun start(context: Context) {
169
+ val intent = Intent(context, NodeForegroundService::class.java)
170
+ context.startForegroundService(intent)
171
+ }
172
+
173
+ fun stop(context: Context) {
174
+ val intent = Intent(context, NodeForegroundService::class.java).setAction(ACTION_STOP)
175
+ context.startService(intent)
176
+ }
177
+ }
178
+ }
179
+
180
+ private data class Quint<A, B, C, D, E>(val first: A, val second: B, val third: C, val fourth: D, val fifth: E)
apps/android/app/src/main/java/ai/openclaw/android/NodeRuntime.kt ADDED
@@ -0,0 +1,1271 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ import android.Manifest
4
+ import android.content.Context
5
+ import android.content.pm.PackageManager
6
+ import android.location.LocationManager
7
+ import android.os.Build
8
+ import android.os.SystemClock
9
+ import androidx.core.content.ContextCompat
10
+ import ai.openclaw.android.chat.ChatController
11
+ import ai.openclaw.android.chat.ChatMessage
12
+ import ai.openclaw.android.chat.ChatPendingToolCall
13
+ import ai.openclaw.android.chat.ChatSessionEntry
14
+ import ai.openclaw.android.chat.OutgoingAttachment
15
+ import ai.openclaw.android.gateway.DeviceAuthStore
16
+ import ai.openclaw.android.gateway.DeviceIdentityStore
17
+ import ai.openclaw.android.gateway.GatewayClientInfo
18
+ import ai.openclaw.android.gateway.GatewayConnectOptions
19
+ import ai.openclaw.android.gateway.GatewayDiscovery
20
+ import ai.openclaw.android.gateway.GatewayEndpoint
21
+ import ai.openclaw.android.gateway.GatewaySession
22
+ import ai.openclaw.android.gateway.GatewayTlsParams
23
+ import ai.openclaw.android.node.CameraCaptureManager
24
+ import ai.openclaw.android.node.LocationCaptureManager
25
+ import ai.openclaw.android.BuildConfig
26
+ import ai.openclaw.android.node.CanvasController
27
+ import ai.openclaw.android.node.ScreenRecordManager
28
+ import ai.openclaw.android.node.SmsManager
29
+ import ai.openclaw.android.protocol.OpenClawCapability
30
+ import ai.openclaw.android.protocol.OpenClawCameraCommand
31
+ import ai.openclaw.android.protocol.OpenClawCanvasA2UIAction
32
+ import ai.openclaw.android.protocol.OpenClawCanvasA2UICommand
33
+ import ai.openclaw.android.protocol.OpenClawCanvasCommand
34
+ import ai.openclaw.android.protocol.OpenClawScreenCommand
35
+ import ai.openclaw.android.protocol.OpenClawLocationCommand
36
+ import ai.openclaw.android.protocol.OpenClawSmsCommand
37
+ import ai.openclaw.android.voice.TalkModeManager
38
+ import ai.openclaw.android.voice.VoiceWakeManager
39
+ import kotlinx.coroutines.CoroutineScope
40
+ import kotlinx.coroutines.Dispatchers
41
+ import kotlinx.coroutines.Job
42
+ import kotlinx.coroutines.SupervisorJob
43
+ import kotlinx.coroutines.TimeoutCancellationException
44
+ import kotlinx.coroutines.delay
45
+ import kotlinx.coroutines.flow.MutableStateFlow
46
+ import kotlinx.coroutines.flow.StateFlow
47
+ import kotlinx.coroutines.flow.asStateFlow
48
+ import kotlinx.coroutines.flow.combine
49
+ import kotlinx.coroutines.flow.collect
50
+ import kotlinx.coroutines.flow.distinctUntilChanged
51
+ import kotlinx.coroutines.launch
52
+ import kotlinx.serialization.json.Json
53
+ import kotlinx.serialization.json.JsonArray
54
+ import kotlinx.serialization.json.JsonElement
55
+ import kotlinx.serialization.json.JsonNull
56
+ import kotlinx.serialization.json.JsonObject
57
+ import kotlinx.serialization.json.JsonPrimitive
58
+ import kotlinx.serialization.json.buildJsonObject
59
+ import java.util.concurrent.atomic.AtomicLong
60
+
61
+ class NodeRuntime(context: Context) {
62
+ private val appContext = context.applicationContext
63
+ private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
64
+
65
+ val prefs = SecurePrefs(appContext)
66
+ private val deviceAuthStore = DeviceAuthStore(prefs)
67
+ val canvas = CanvasController()
68
+ val camera = CameraCaptureManager(appContext)
69
+ val location = LocationCaptureManager(appContext)
70
+ val screenRecorder = ScreenRecordManager(appContext)
71
+ val sms = SmsManager(appContext)
72
+ private val json = Json { ignoreUnknownKeys = true }
73
+
74
+ private val externalAudioCaptureActive = MutableStateFlow(false)
75
+
76
+ private val voiceWake: VoiceWakeManager by lazy {
77
+ VoiceWakeManager(
78
+ context = appContext,
79
+ scope = scope,
80
+ onCommand = { command ->
81
+ nodeSession.sendNodeEvent(
82
+ event = "agent.request",
83
+ payloadJson =
84
+ buildJsonObject {
85
+ put("message", JsonPrimitive(command))
86
+ put("sessionKey", JsonPrimitive(resolveMainSessionKey()))
87
+ put("thinking", JsonPrimitive(chatThinkingLevel.value))
88
+ put("deliver", JsonPrimitive(false))
89
+ }.toString(),
90
+ )
91
+ },
92
+ )
93
+ }
94
+
95
+ val voiceWakeIsListening: StateFlow<Boolean>
96
+ get() = voiceWake.isListening
97
+
98
+ val voiceWakeStatusText: StateFlow<String>
99
+ get() = voiceWake.statusText
100
+
101
+ val talkStatusText: StateFlow<String>
102
+ get() = talkMode.statusText
103
+
104
+ val talkIsListening: StateFlow<Boolean>
105
+ get() = talkMode.isListening
106
+
107
+ val talkIsSpeaking: StateFlow<Boolean>
108
+ get() = talkMode.isSpeaking
109
+
110
+ private val discovery = GatewayDiscovery(appContext, scope = scope)
111
+ val gateways: StateFlow<List<GatewayEndpoint>> = discovery.gateways
112
+ val discoveryStatusText: StateFlow<String> = discovery.statusText
113
+
114
+ private val identityStore = DeviceIdentityStore(appContext)
115
+
116
+ private val _isConnected = MutableStateFlow(false)
117
+ val isConnected: StateFlow<Boolean> = _isConnected.asStateFlow()
118
+
119
+ private val _statusText = MutableStateFlow("Offline")
120
+ val statusText: StateFlow<String> = _statusText.asStateFlow()
121
+
122
+ private val _mainSessionKey = MutableStateFlow("main")
123
+ val mainSessionKey: StateFlow<String> = _mainSessionKey.asStateFlow()
124
+
125
+ private val cameraHudSeq = AtomicLong(0)
126
+ private val _cameraHud = MutableStateFlow<CameraHudState?>(null)
127
+ val cameraHud: StateFlow<CameraHudState?> = _cameraHud.asStateFlow()
128
+
129
+ private val _cameraFlashToken = MutableStateFlow(0L)
130
+ val cameraFlashToken: StateFlow<Long> = _cameraFlashToken.asStateFlow()
131
+
132
+ private val _screenRecordActive = MutableStateFlow(false)
133
+ val screenRecordActive: StateFlow<Boolean> = _screenRecordActive.asStateFlow()
134
+
135
+ private val _serverName = MutableStateFlow<String?>(null)
136
+ val serverName: StateFlow<String?> = _serverName.asStateFlow()
137
+
138
+ private val _remoteAddress = MutableStateFlow<String?>(null)
139
+ val remoteAddress: StateFlow<String?> = _remoteAddress.asStateFlow()
140
+
141
+ private val _seamColorArgb = MutableStateFlow(DEFAULT_SEAM_COLOR_ARGB)
142
+ val seamColorArgb: StateFlow<Long> = _seamColorArgb.asStateFlow()
143
+
144
+ private val _isForeground = MutableStateFlow(true)
145
+ val isForeground: StateFlow<Boolean> = _isForeground.asStateFlow()
146
+
147
+ private var lastAutoA2uiUrl: String? = null
148
+ private var operatorConnected = false
149
+ private var nodeConnected = false
150
+ private var operatorStatusText: String = "Offline"
151
+ private var nodeStatusText: String = "Offline"
152
+ private var connectedEndpoint: GatewayEndpoint? = null
153
+
154
+ private val operatorSession =
155
+ GatewaySession(
156
+ scope = scope,
157
+ identityStore = identityStore,
158
+ deviceAuthStore = deviceAuthStore,
159
+ onConnected = { name, remote, mainSessionKey ->
160
+ operatorConnected = true
161
+ operatorStatusText = "Connected"
162
+ _serverName.value = name
163
+ _remoteAddress.value = remote
164
+ _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
165
+ applyMainSessionKey(mainSessionKey)
166
+ updateStatus()
167
+ scope.launch { refreshBrandingFromGateway() }
168
+ scope.launch { refreshWakeWordsFromGateway() }
169
+ },
170
+ onDisconnected = { message ->
171
+ operatorConnected = false
172
+ operatorStatusText = message
173
+ _serverName.value = null
174
+ _remoteAddress.value = null
175
+ _seamColorArgb.value = DEFAULT_SEAM_COLOR_ARGB
176
+ if (!isCanonicalMainSessionKey(_mainSessionKey.value)) {
177
+ _mainSessionKey.value = "main"
178
+ }
179
+ val mainKey = resolveMainSessionKey()
180
+ talkMode.setMainSessionKey(mainKey)
181
+ chat.applyMainSessionKey(mainKey)
182
+ chat.onDisconnected(message)
183
+ updateStatus()
184
+ },
185
+ onEvent = { event, payloadJson ->
186
+ handleGatewayEvent(event, payloadJson)
187
+ },
188
+ )
189
+
190
+ private val nodeSession =
191
+ GatewaySession(
192
+ scope = scope,
193
+ identityStore = identityStore,
194
+ deviceAuthStore = deviceAuthStore,
195
+ onConnected = { _, _, _ ->
196
+ nodeConnected = true
197
+ nodeStatusText = "Connected"
198
+ updateStatus()
199
+ maybeNavigateToA2uiOnConnect()
200
+ },
201
+ onDisconnected = { message ->
202
+ nodeConnected = false
203
+ nodeStatusText = message
204
+ updateStatus()
205
+ showLocalCanvasOnDisconnect()
206
+ },
207
+ onEvent = { _, _ -> },
208
+ onInvoke = { req ->
209
+ handleInvoke(req.command, req.paramsJson)
210
+ },
211
+ onTlsFingerprint = { stableId, fingerprint ->
212
+ prefs.saveGatewayTlsFingerprint(stableId, fingerprint)
213
+ },
214
+ )
215
+
216
+ private val chat: ChatController =
217
+ ChatController(
218
+ scope = scope,
219
+ session = operatorSession,
220
+ json = json,
221
+ supportsChatSubscribe = false,
222
+ )
223
+ private val talkMode: TalkModeManager by lazy {
224
+ TalkModeManager(
225
+ context = appContext,
226
+ scope = scope,
227
+ session = operatorSession,
228
+ supportsChatSubscribe = false,
229
+ isConnected = { operatorConnected },
230
+ )
231
+ }
232
+
233
+ private fun applyMainSessionKey(candidate: String?) {
234
+ val trimmed = candidate?.trim().orEmpty()
235
+ if (trimmed.isEmpty()) return
236
+ if (isCanonicalMainSessionKey(_mainSessionKey.value)) return
237
+ if (_mainSessionKey.value == trimmed) return
238
+ _mainSessionKey.value = trimmed
239
+ talkMode.setMainSessionKey(trimmed)
240
+ chat.applyMainSessionKey(trimmed)
241
+ }
242
+
243
+ private fun updateStatus() {
244
+ _isConnected.value = operatorConnected
245
+ _statusText.value =
246
+ when {
247
+ operatorConnected && nodeConnected -> "Connected"
248
+ operatorConnected && !nodeConnected -> "Connected (node offline)"
249
+ !operatorConnected && nodeConnected -> "Connected (operator offline)"
250
+ operatorStatusText.isNotBlank() && operatorStatusText != "Offline" -> operatorStatusText
251
+ else -> nodeStatusText
252
+ }
253
+ }
254
+
255
+ private fun resolveMainSessionKey(): String {
256
+ val trimmed = _mainSessionKey.value.trim()
257
+ return if (trimmed.isEmpty()) "main" else trimmed
258
+ }
259
+
260
+ private fun maybeNavigateToA2uiOnConnect() {
261
+ val a2uiUrl = resolveA2uiHostUrl() ?: return
262
+ val current = canvas.currentUrl()?.trim().orEmpty()
263
+ if (current.isEmpty() || current == lastAutoA2uiUrl) {
264
+ lastAutoA2uiUrl = a2uiUrl
265
+ canvas.navigate(a2uiUrl)
266
+ }
267
+ }
268
+
269
+ private fun showLocalCanvasOnDisconnect() {
270
+ lastAutoA2uiUrl = null
271
+ canvas.navigate("")
272
+ }
273
+
274
+ val instanceId: StateFlow<String> = prefs.instanceId
275
+ val displayName: StateFlow<String> = prefs.displayName
276
+ val cameraEnabled: StateFlow<Boolean> = prefs.cameraEnabled
277
+ val locationMode: StateFlow<LocationMode> = prefs.locationMode
278
+ val locationPreciseEnabled: StateFlow<Boolean> = prefs.locationPreciseEnabled
279
+ val preventSleep: StateFlow<Boolean> = prefs.preventSleep
280
+ val wakeWords: StateFlow<List<String>> = prefs.wakeWords
281
+ val voiceWakeMode: StateFlow<VoiceWakeMode> = prefs.voiceWakeMode
282
+ val talkEnabled: StateFlow<Boolean> = prefs.talkEnabled
283
+ val manualEnabled: StateFlow<Boolean> = prefs.manualEnabled
284
+ val manualHost: StateFlow<String> = prefs.manualHost
285
+ val manualPort: StateFlow<Int> = prefs.manualPort
286
+ val manualTls: StateFlow<Boolean> = prefs.manualTls
287
+ val lastDiscoveredStableId: StateFlow<String> = prefs.lastDiscoveredStableId
288
+ val canvasDebugStatusEnabled: StateFlow<Boolean> = prefs.canvasDebugStatusEnabled
289
+
290
+ private var didAutoConnect = false
291
+ private var suppressWakeWordsSync = false
292
+ private var wakeWordsSyncJob: Job? = null
293
+
294
+ val chatSessionKey: StateFlow<String> = chat.sessionKey
295
+ val chatSessionId: StateFlow<String?> = chat.sessionId
296
+ val chatMessages: StateFlow<List<ChatMessage>> = chat.messages
297
+ val chatError: StateFlow<String?> = chat.errorText
298
+ val chatHealthOk: StateFlow<Boolean> = chat.healthOk
299
+ val chatThinkingLevel: StateFlow<String> = chat.thinkingLevel
300
+ val chatStreamingAssistantText: StateFlow<String?> = chat.streamingAssistantText
301
+ val chatPendingToolCalls: StateFlow<List<ChatPendingToolCall>> = chat.pendingToolCalls
302
+ val chatSessions: StateFlow<List<ChatSessionEntry>> = chat.sessions
303
+ val pendingRunCount: StateFlow<Int> = chat.pendingRunCount
304
+
305
+ init {
306
+ scope.launch {
307
+ combine(
308
+ voiceWakeMode,
309
+ isForeground,
310
+ externalAudioCaptureActive,
311
+ wakeWords,
312
+ ) { mode, foreground, externalAudio, words ->
313
+ Quad(mode, foreground, externalAudio, words)
314
+ }.distinctUntilChanged()
315
+ .collect { (mode, foreground, externalAudio, words) ->
316
+ voiceWake.setTriggerWords(words)
317
+
318
+ val shouldListen =
319
+ when (mode) {
320
+ VoiceWakeMode.Off -> false
321
+ VoiceWakeMode.Foreground -> foreground
322
+ VoiceWakeMode.Always -> true
323
+ } && !externalAudio
324
+
325
+ if (!shouldListen) {
326
+ voiceWake.stop(statusText = if (mode == VoiceWakeMode.Off) "Off" else "Paused")
327
+ return@collect
328
+ }
329
+
330
+ if (!hasRecordAudioPermission()) {
331
+ voiceWake.stop(statusText = "Microphone permission required")
332
+ return@collect
333
+ }
334
+
335
+ voiceWake.start()
336
+ }
337
+ }
338
+
339
+ scope.launch {
340
+ talkEnabled.collect { enabled ->
341
+ talkMode.setEnabled(enabled)
342
+ externalAudioCaptureActive.value = enabled
343
+ }
344
+ }
345
+
346
+ scope.launch(Dispatchers.Default) {
347
+ gateways.collect { list ->
348
+ if (list.isNotEmpty()) {
349
+ // Persist the last discovered gateway (best-effort UX parity with iOS).
350
+ prefs.setLastDiscoveredStableId(list.last().stableId)
351
+ }
352
+
353
+ if (didAutoConnect) return@collect
354
+ if (_isConnected.value) return@collect
355
+
356
+ if (manualEnabled.value) {
357
+ val host = manualHost.value.trim()
358
+ val port = manualPort.value
359
+ if (host.isNotEmpty() && port in 1..65535) {
360
+ didAutoConnect = true
361
+ connect(GatewayEndpoint.manual(host = host, port = port))
362
+ }
363
+ return@collect
364
+ }
365
+
366
+ val targetStableId = lastDiscoveredStableId.value.trim()
367
+ if (targetStableId.isEmpty()) return@collect
368
+ val target = list.firstOrNull { it.stableId == targetStableId } ?: return@collect
369
+ didAutoConnect = true
370
+ connect(target)
371
+ }
372
+ }
373
+
374
+ scope.launch {
375
+ combine(
376
+ canvasDebugStatusEnabled,
377
+ statusText,
378
+ serverName,
379
+ remoteAddress,
380
+ ) { debugEnabled, status, server, remote ->
381
+ Quad(debugEnabled, status, server, remote)
382
+ }.distinctUntilChanged()
383
+ .collect { (debugEnabled, status, server, remote) ->
384
+ canvas.setDebugStatusEnabled(debugEnabled)
385
+ if (!debugEnabled) return@collect
386
+ canvas.setDebugStatus(status, server ?: remote)
387
+ }
388
+ }
389
+ }
390
+
391
+ fun setForeground(value: Boolean) {
392
+ _isForeground.value = value
393
+ }
394
+
395
+ fun setDisplayName(value: String) {
396
+ prefs.setDisplayName(value)
397
+ }
398
+
399
+ fun setCameraEnabled(value: Boolean) {
400
+ prefs.setCameraEnabled(value)
401
+ }
402
+
403
+ fun setLocationMode(mode: LocationMode) {
404
+ prefs.setLocationMode(mode)
405
+ }
406
+
407
+ fun setLocationPreciseEnabled(value: Boolean) {
408
+ prefs.setLocationPreciseEnabled(value)
409
+ }
410
+
411
+ fun setPreventSleep(value: Boolean) {
412
+ prefs.setPreventSleep(value)
413
+ }
414
+
415
+ fun setManualEnabled(value: Boolean) {
416
+ prefs.setManualEnabled(value)
417
+ }
418
+
419
+ fun setManualHost(value: String) {
420
+ prefs.setManualHost(value)
421
+ }
422
+
423
+ fun setManualPort(value: Int) {
424
+ prefs.setManualPort(value)
425
+ }
426
+
427
+ fun setManualTls(value: Boolean) {
428
+ prefs.setManualTls(value)
429
+ }
430
+
431
+ fun setCanvasDebugStatusEnabled(value: Boolean) {
432
+ prefs.setCanvasDebugStatusEnabled(value)
433
+ }
434
+
435
+ fun setWakeWords(words: List<String>) {
436
+ prefs.setWakeWords(words)
437
+ scheduleWakeWordsSyncIfNeeded()
438
+ }
439
+
440
+ fun resetWakeWordsDefaults() {
441
+ setWakeWords(SecurePrefs.defaultWakeWords)
442
+ }
443
+
444
+ fun setVoiceWakeMode(mode: VoiceWakeMode) {
445
+ prefs.setVoiceWakeMode(mode)
446
+ }
447
+
448
+ fun setTalkEnabled(value: Boolean) {
449
+ prefs.setTalkEnabled(value)
450
+ }
451
+
452
+ private fun buildInvokeCommands(): List<String> =
453
+ buildList {
454
+ add(OpenClawCanvasCommand.Present.rawValue)
455
+ add(OpenClawCanvasCommand.Hide.rawValue)
456
+ add(OpenClawCanvasCommand.Navigate.rawValue)
457
+ add(OpenClawCanvasCommand.Eval.rawValue)
458
+ add(OpenClawCanvasCommand.Snapshot.rawValue)
459
+ add(OpenClawCanvasA2UICommand.Push.rawValue)
460
+ add(OpenClawCanvasA2UICommand.PushJSONL.rawValue)
461
+ add(OpenClawCanvasA2UICommand.Reset.rawValue)
462
+ add(OpenClawScreenCommand.Record.rawValue)
463
+ if (cameraEnabled.value) {
464
+ add(OpenClawCameraCommand.Snap.rawValue)
465
+ add(OpenClawCameraCommand.Clip.rawValue)
466
+ }
467
+ if (locationMode.value != LocationMode.Off) {
468
+ add(OpenClawLocationCommand.Get.rawValue)
469
+ }
470
+ if (sms.canSendSms()) {
471
+ add(OpenClawSmsCommand.Send.rawValue)
472
+ }
473
+ }
474
+
475
+ private fun buildCapabilities(): List<String> =
476
+ buildList {
477
+ add(OpenClawCapability.Canvas.rawValue)
478
+ add(OpenClawCapability.Screen.rawValue)
479
+ if (cameraEnabled.value) add(OpenClawCapability.Camera.rawValue)
480
+ if (sms.canSendSms()) add(OpenClawCapability.Sms.rawValue)
481
+ if (voiceWakeMode.value != VoiceWakeMode.Off && hasRecordAudioPermission()) {
482
+ add(OpenClawCapability.VoiceWake.rawValue)
483
+ }
484
+ if (locationMode.value != LocationMode.Off) {
485
+ add(OpenClawCapability.Location.rawValue)
486
+ }
487
+ }
488
+
489
+ private fun resolvedVersionName(): String {
490
+ val versionName = BuildConfig.VERSION_NAME.trim().ifEmpty { "dev" }
491
+ return if (BuildConfig.DEBUG && !versionName.contains("dev", ignoreCase = true)) {
492
+ "$versionName-dev"
493
+ } else {
494
+ versionName
495
+ }
496
+ }
497
+
498
+ private fun resolveModelIdentifier(): String? {
499
+ return listOfNotNull(Build.MANUFACTURER, Build.MODEL)
500
+ .joinToString(" ")
501
+ .trim()
502
+ .ifEmpty { null }
503
+ }
504
+
505
+ private fun buildUserAgent(): String {
506
+ val version = resolvedVersionName()
507
+ val release = Build.VERSION.RELEASE?.trim().orEmpty()
508
+ val releaseLabel = if (release.isEmpty()) "unknown" else release
509
+ return "OpenClawAndroid/$version (Android $releaseLabel; SDK ${Build.VERSION.SDK_INT})"
510
+ }
511
+
512
+ private fun buildClientInfo(clientId: String, clientMode: String): GatewayClientInfo {
513
+ return GatewayClientInfo(
514
+ id = clientId,
515
+ displayName = displayName.value,
516
+ version = resolvedVersionName(),
517
+ platform = "android",
518
+ mode = clientMode,
519
+ instanceId = instanceId.value,
520
+ deviceFamily = "Android",
521
+ modelIdentifier = resolveModelIdentifier(),
522
+ )
523
+ }
524
+
525
+ private fun buildNodeConnectOptions(): GatewayConnectOptions {
526
+ return GatewayConnectOptions(
527
+ role = "node",
528
+ scopes = emptyList(),
529
+ caps = buildCapabilities(),
530
+ commands = buildInvokeCommands(),
531
+ permissions = emptyMap(),
532
+ client = buildClientInfo(clientId = "openclaw-android", clientMode = "node"),
533
+ userAgent = buildUserAgent(),
534
+ )
535
+ }
536
+
537
+ private fun buildOperatorConnectOptions(): GatewayConnectOptions {
538
+ return GatewayConnectOptions(
539
+ role = "operator",
540
+ scopes = emptyList(),
541
+ caps = emptyList(),
542
+ commands = emptyList(),
543
+ permissions = emptyMap(),
544
+ client = buildClientInfo(clientId = "openclaw-control-ui", clientMode = "ui"),
545
+ userAgent = buildUserAgent(),
546
+ )
547
+ }
548
+
549
+ fun refreshGatewayConnection() {
550
+ val endpoint = connectedEndpoint ?: return
551
+ val token = prefs.loadGatewayToken()
552
+ val password = prefs.loadGatewayPassword()
553
+ val tls = resolveTlsParams(endpoint)
554
+ operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls)
555
+ nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls)
556
+ operatorSession.reconnect()
557
+ nodeSession.reconnect()
558
+ }
559
+
560
+ fun connect(endpoint: GatewayEndpoint) {
561
+ connectedEndpoint = endpoint
562
+ operatorStatusText = "Connecting…"
563
+ nodeStatusText = "Connecting…"
564
+ updateStatus()
565
+ val token = prefs.loadGatewayToken()
566
+ val password = prefs.loadGatewayPassword()
567
+ val tls = resolveTlsParams(endpoint)
568
+ operatorSession.connect(endpoint, token, password, buildOperatorConnectOptions(), tls)
569
+ nodeSession.connect(endpoint, token, password, buildNodeConnectOptions(), tls)
570
+ }
571
+
572
+ private fun hasRecordAudioPermission(): Boolean {
573
+ return (
574
+ ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) ==
575
+ PackageManager.PERMISSION_GRANTED
576
+ )
577
+ }
578
+
579
+ private fun hasFineLocationPermission(): Boolean {
580
+ return (
581
+ ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) ==
582
+ PackageManager.PERMISSION_GRANTED
583
+ )
584
+ }
585
+
586
+ private fun hasCoarseLocationPermission(): Boolean {
587
+ return (
588
+ ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) ==
589
+ PackageManager.PERMISSION_GRANTED
590
+ )
591
+ }
592
+
593
+ private fun hasBackgroundLocationPermission(): Boolean {
594
+ return (
595
+ ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ==
596
+ PackageManager.PERMISSION_GRANTED
597
+ )
598
+ }
599
+
600
+ fun connectManual() {
601
+ val host = manualHost.value.trim()
602
+ val port = manualPort.value
603
+ if (host.isEmpty() || port <= 0 || port > 65535) {
604
+ _statusText.value = "Failed: invalid manual host/port"
605
+ return
606
+ }
607
+ connect(GatewayEndpoint.manual(host = host, port = port))
608
+ }
609
+
610
+ fun disconnect() {
611
+ connectedEndpoint = null
612
+ operatorSession.disconnect()
613
+ nodeSession.disconnect()
614
+ }
615
+
616
+ private fun resolveTlsParams(endpoint: GatewayEndpoint): GatewayTlsParams? {
617
+ val stored = prefs.loadGatewayTlsFingerprint(endpoint.stableId)
618
+ val hinted = endpoint.tlsEnabled || !endpoint.tlsFingerprintSha256.isNullOrBlank()
619
+ val manual = endpoint.stableId.startsWith("manual|")
620
+
621
+ if (manual) {
622
+ if (!manualTls.value) return null
623
+ return GatewayTlsParams(
624
+ required = true,
625
+ expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
626
+ allowTOFU = stored == null,
627
+ stableId = endpoint.stableId,
628
+ )
629
+ }
630
+
631
+ if (hinted) {
632
+ return GatewayTlsParams(
633
+ required = true,
634
+ expectedFingerprint = endpoint.tlsFingerprintSha256 ?: stored,
635
+ allowTOFU = stored == null,
636
+ stableId = endpoint.stableId,
637
+ )
638
+ }
639
+
640
+ if (!stored.isNullOrBlank()) {
641
+ return GatewayTlsParams(
642
+ required = true,
643
+ expectedFingerprint = stored,
644
+ allowTOFU = false,
645
+ stableId = endpoint.stableId,
646
+ )
647
+ }
648
+
649
+ return null
650
+ }
651
+
652
+ fun handleCanvasA2UIActionFromWebView(payloadJson: String) {
653
+ scope.launch {
654
+ val trimmed = payloadJson.trim()
655
+ if (trimmed.isEmpty()) return@launch
656
+
657
+ val root =
658
+ try {
659
+ json.parseToJsonElement(trimmed).asObjectOrNull() ?: return@launch
660
+ } catch (_: Throwable) {
661
+ return@launch
662
+ }
663
+
664
+ val userActionObj = (root["userAction"] as? JsonObject) ?: root
665
+ val actionId = (userActionObj["id"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty {
666
+ java.util.UUID.randomUUID().toString()
667
+ }
668
+ val name = OpenClawCanvasA2UIAction.extractActionName(userActionObj) ?: return@launch
669
+
670
+ val surfaceId =
671
+ (userActionObj["surfaceId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "main" }
672
+ val sourceComponentId =
673
+ (userActionObj["sourceComponentId"] as? JsonPrimitive)?.content?.trim().orEmpty().ifEmpty { "-" }
674
+ val contextJson = (userActionObj["context"] as? JsonObject)?.toString()
675
+
676
+ val sessionKey = resolveMainSessionKey()
677
+ val message =
678
+ OpenClawCanvasA2UIAction.formatAgentMessage(
679
+ actionName = name,
680
+ sessionKey = sessionKey,
681
+ surfaceId = surfaceId,
682
+ sourceComponentId = sourceComponentId,
683
+ host = displayName.value,
684
+ instanceId = instanceId.value.lowercase(),
685
+ contextJson = contextJson,
686
+ )
687
+
688
+ val connected = nodeConnected
689
+ var error: String? = null
690
+ if (connected) {
691
+ try {
692
+ nodeSession.sendNodeEvent(
693
+ event = "agent.request",
694
+ payloadJson =
695
+ buildJsonObject {
696
+ put("message", JsonPrimitive(message))
697
+ put("sessionKey", JsonPrimitive(sessionKey))
698
+ put("thinking", JsonPrimitive("low"))
699
+ put("deliver", JsonPrimitive(false))
700
+ put("key", JsonPrimitive(actionId))
701
+ }.toString(),
702
+ )
703
+ } catch (e: Throwable) {
704
+ error = e.message ?: "send failed"
705
+ }
706
+ } else {
707
+ error = "gateway not connected"
708
+ }
709
+
710
+ try {
711
+ canvas.eval(
712
+ OpenClawCanvasA2UIAction.jsDispatchA2UIActionStatus(
713
+ actionId = actionId,
714
+ ok = connected && error == null,
715
+ error = error,
716
+ ),
717
+ )
718
+ } catch (_: Throwable) {
719
+ // ignore
720
+ }
721
+ }
722
+ }
723
+
724
+ fun loadChat(sessionKey: String) {
725
+ val key = sessionKey.trim().ifEmpty { resolveMainSessionKey() }
726
+ chat.load(key)
727
+ }
728
+
729
+ fun refreshChat() {
730
+ chat.refresh()
731
+ }
732
+
733
+ fun refreshChatSessions(limit: Int? = null) {
734
+ chat.refreshSessions(limit = limit)
735
+ }
736
+
737
+ fun setChatThinkingLevel(level: String) {
738
+ chat.setThinkingLevel(level)
739
+ }
740
+
741
+ fun switchChatSession(sessionKey: String) {
742
+ chat.switchSession(sessionKey)
743
+ }
744
+
745
+ fun abortChat() {
746
+ chat.abort()
747
+ }
748
+
749
+ fun sendChat(message: String, thinking: String, attachments: List<OutgoingAttachment>) {
750
+ chat.sendMessage(message = message, thinkingLevel = thinking, attachments = attachments)
751
+ }
752
+
753
+ private fun handleGatewayEvent(event: String, payloadJson: String?) {
754
+ if (event == "voicewake.changed") {
755
+ if (payloadJson.isNullOrBlank()) return
756
+ try {
757
+ val payload = json.parseToJsonElement(payloadJson).asObjectOrNull() ?: return
758
+ val array = payload["triggers"] as? JsonArray ?: return
759
+ val triggers = array.mapNotNull { it.asStringOrNull() }
760
+ applyWakeWordsFromGateway(triggers)
761
+ } catch (_: Throwable) {
762
+ // ignore
763
+ }
764
+ return
765
+ }
766
+
767
+ talkMode.handleGatewayEvent(event, payloadJson)
768
+ chat.handleGatewayEvent(event, payloadJson)
769
+ }
770
+
771
+ private fun applyWakeWordsFromGateway(words: List<String>) {
772
+ suppressWakeWordsSync = true
773
+ prefs.setWakeWords(words)
774
+ suppressWakeWordsSync = false
775
+ }
776
+
777
+ private fun scheduleWakeWordsSyncIfNeeded() {
778
+ if (suppressWakeWordsSync) return
779
+ if (!_isConnected.value) return
780
+
781
+ val snapshot = prefs.wakeWords.value
782
+ wakeWordsSyncJob?.cancel()
783
+ wakeWordsSyncJob =
784
+ scope.launch {
785
+ delay(650)
786
+ val jsonList = snapshot.joinToString(separator = ",") { it.toJsonString() }
787
+ val params = """{"triggers":[$jsonList]}"""
788
+ try {
789
+ operatorSession.request("voicewake.set", params)
790
+ } catch (_: Throwable) {
791
+ // ignore
792
+ }
793
+ }
794
+ }
795
+
796
+ private suspend fun refreshWakeWordsFromGateway() {
797
+ if (!_isConnected.value) return
798
+ try {
799
+ val res = operatorSession.request("voicewake.get", "{}")
800
+ val payload = json.parseToJsonElement(res).asObjectOrNull() ?: return
801
+ val array = payload["triggers"] as? JsonArray ?: return
802
+ val triggers = array.mapNotNull { it.asStringOrNull() }
803
+ applyWakeWordsFromGateway(triggers)
804
+ } catch (_: Throwable) {
805
+ // ignore
806
+ }
807
+ }
808
+
809
+ private suspend fun refreshBrandingFromGateway() {
810
+ if (!_isConnected.value) return
811
+ try {
812
+ val res = operatorSession.request("config.get", "{}")
813
+ val root = json.parseToJsonElement(res).asObjectOrNull()
814
+ val config = root?.get("config").asObjectOrNull()
815
+ val ui = config?.get("ui").asObjectOrNull()
816
+ val raw = ui?.get("seamColor").asStringOrNull()?.trim()
817
+ val sessionCfg = config?.get("session").asObjectOrNull()
818
+ val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull())
819
+ applyMainSessionKey(mainKey)
820
+
821
+ val parsed = parseHexColorArgb(raw)
822
+ _seamColorArgb.value = parsed ?: DEFAULT_SEAM_COLOR_ARGB
823
+ } catch (_: Throwable) {
824
+ // ignore
825
+ }
826
+ }
827
+
828
+ private suspend fun handleInvoke(command: String, paramsJson: String?): GatewaySession.InvokeResult {
829
+ if (
830
+ command.startsWith(OpenClawCanvasCommand.NamespacePrefix) ||
831
+ command.startsWith(OpenClawCanvasA2UICommand.NamespacePrefix) ||
832
+ command.startsWith(OpenClawCameraCommand.NamespacePrefix) ||
833
+ command.startsWith(OpenClawScreenCommand.NamespacePrefix)
834
+ ) {
835
+ if (!isForeground.value) {
836
+ return GatewaySession.InvokeResult.error(
837
+ code = "NODE_BACKGROUND_UNAVAILABLE",
838
+ message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground",
839
+ )
840
+ }
841
+ }
842
+ if (command.startsWith(OpenClawCameraCommand.NamespacePrefix) && !cameraEnabled.value) {
843
+ return GatewaySession.InvokeResult.error(
844
+ code = "CAMERA_DISABLED",
845
+ message = "CAMERA_DISABLED: enable Camera in Settings",
846
+ )
847
+ }
848
+ if (command.startsWith(OpenClawLocationCommand.NamespacePrefix) &&
849
+ locationMode.value == LocationMode.Off
850
+ ) {
851
+ return GatewaySession.InvokeResult.error(
852
+ code = "LOCATION_DISABLED",
853
+ message = "LOCATION_DISABLED: enable Location in Settings",
854
+ )
855
+ }
856
+
857
+ return when (command) {
858
+ OpenClawCanvasCommand.Present.rawValue -> {
859
+ val url = CanvasController.parseNavigateUrl(paramsJson)
860
+ canvas.navigate(url)
861
+ GatewaySession.InvokeResult.ok(null)
862
+ }
863
+ OpenClawCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null)
864
+ OpenClawCanvasCommand.Navigate.rawValue -> {
865
+ val url = CanvasController.parseNavigateUrl(paramsJson)
866
+ canvas.navigate(url)
867
+ GatewaySession.InvokeResult.ok(null)
868
+ }
869
+ OpenClawCanvasCommand.Eval.rawValue -> {
870
+ val js =
871
+ CanvasController.parseEvalJs(paramsJson)
872
+ ?: return GatewaySession.InvokeResult.error(
873
+ code = "INVALID_REQUEST",
874
+ message = "INVALID_REQUEST: javaScript required",
875
+ )
876
+ val result =
877
+ try {
878
+ canvas.eval(js)
879
+ } catch (err: Throwable) {
880
+ return GatewaySession.InvokeResult.error(
881
+ code = "NODE_BACKGROUND_UNAVAILABLE",
882
+ message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
883
+ )
884
+ }
885
+ GatewaySession.InvokeResult.ok("""{"result":${result.toJsonString()}}""")
886
+ }
887
+ OpenClawCanvasCommand.Snapshot.rawValue -> {
888
+ val snapshotParams = CanvasController.parseSnapshotParams(paramsJson)
889
+ val base64 =
890
+ try {
891
+ canvas.snapshotBase64(
892
+ format = snapshotParams.format,
893
+ quality = snapshotParams.quality,
894
+ maxWidth = snapshotParams.maxWidth,
895
+ )
896
+ } catch (err: Throwable) {
897
+ return GatewaySession.InvokeResult.error(
898
+ code = "NODE_BACKGROUND_UNAVAILABLE",
899
+ message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable",
900
+ )
901
+ }
902
+ GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""")
903
+ }
904
+ OpenClawCanvasA2UICommand.Reset.rawValue -> {
905
+ val a2uiUrl = resolveA2uiHostUrl()
906
+ ?: return GatewaySession.InvokeResult.error(
907
+ code = "A2UI_HOST_NOT_CONFIGURED",
908
+ message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
909
+ )
910
+ val ready = ensureA2uiReady(a2uiUrl)
911
+ if (!ready) {
912
+ return GatewaySession.InvokeResult.error(
913
+ code = "A2UI_HOST_UNAVAILABLE",
914
+ message = "A2UI host not reachable",
915
+ )
916
+ }
917
+ val res = canvas.eval(a2uiResetJS)
918
+ GatewaySession.InvokeResult.ok(res)
919
+ }
920
+ OpenClawCanvasA2UICommand.Push.rawValue, OpenClawCanvasA2UICommand.PushJSONL.rawValue -> {
921
+ val messages =
922
+ try {
923
+ decodeA2uiMessages(command, paramsJson)
924
+ } catch (err: Throwable) {
925
+ return GatewaySession.InvokeResult.error(code = "INVALID_REQUEST", message = err.message ?: "invalid A2UI payload")
926
+ }
927
+ val a2uiUrl = resolveA2uiHostUrl()
928
+ ?: return GatewaySession.InvokeResult.error(
929
+ code = "A2UI_HOST_NOT_CONFIGURED",
930
+ message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host",
931
+ )
932
+ val ready = ensureA2uiReady(a2uiUrl)
933
+ if (!ready) {
934
+ return GatewaySession.InvokeResult.error(
935
+ code = "A2UI_HOST_UNAVAILABLE",
936
+ message = "A2UI host not reachable",
937
+ )
938
+ }
939
+ val js = a2uiApplyMessagesJS(messages)
940
+ val res = canvas.eval(js)
941
+ GatewaySession.InvokeResult.ok(res)
942
+ }
943
+ OpenClawCameraCommand.Snap.rawValue -> {
944
+ showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo)
945
+ triggerCameraFlash()
946
+ val res =
947
+ try {
948
+ camera.snap(paramsJson)
949
+ } catch (err: Throwable) {
950
+ val (code, message) = invokeErrorFromThrowable(err)
951
+ showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200)
952
+ return GatewaySession.InvokeResult.error(code = code, message = message)
953
+ }
954
+ showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600)
955
+ GatewaySession.InvokeResult.ok(res.payloadJson)
956
+ }
957
+ OpenClawCameraCommand.Clip.rawValue -> {
958
+ val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false
959
+ if (includeAudio) externalAudioCaptureActive.value = true
960
+ try {
961
+ showCameraHud(message = "Recording…", kind = CameraHudKind.Recording)
962
+ val res =
963
+ try {
964
+ camera.clip(paramsJson)
965
+ } catch (err: Throwable) {
966
+ val (code, message) = invokeErrorFromThrowable(err)
967
+ showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400)
968
+ return GatewaySession.InvokeResult.error(code = code, message = message)
969
+ }
970
+ showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800)
971
+ GatewaySession.InvokeResult.ok(res.payloadJson)
972
+ } finally {
973
+ if (includeAudio) externalAudioCaptureActive.value = false
974
+ }
975
+ }
976
+ OpenClawLocationCommand.Get.rawValue -> {
977
+ val mode = locationMode.value
978
+ if (!isForeground.value && mode != LocationMode.Always) {
979
+ return GatewaySession.InvokeResult.error(
980
+ code = "LOCATION_BACKGROUND_UNAVAILABLE",
981
+ message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always",
982
+ )
983
+ }
984
+ if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) {
985
+ return GatewaySession.InvokeResult.error(
986
+ code = "LOCATION_PERMISSION_REQUIRED",
987
+ message = "LOCATION_PERMISSION_REQUIRED: grant Location permission",
988
+ )
989
+ }
990
+ if (!isForeground.value && mode == LocationMode.Always && !hasBackgroundLocationPermission()) {
991
+ return GatewaySession.InvokeResult.error(
992
+ code = "LOCATION_PERMISSION_REQUIRED",
993
+ message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings",
994
+ )
995
+ }
996
+ val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson)
997
+ val preciseEnabled = locationPreciseEnabled.value
998
+ val accuracy =
999
+ when (desiredAccuracy) {
1000
+ "precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
1001
+ "coarse" -> "coarse"
1002
+ else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced"
1003
+ }
1004
+ val providers =
1005
+ when (accuracy) {
1006
+ "precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER)
1007
+ "coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
1008
+ else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER)
1009
+ }
1010
+ try {
1011
+ val payload =
1012
+ location.getLocation(
1013
+ desiredProviders = providers,
1014
+ maxAgeMs = maxAgeMs,
1015
+ timeoutMs = timeoutMs,
1016
+ isPrecise = accuracy == "precise",
1017
+ )
1018
+ GatewaySession.InvokeResult.ok(payload.payloadJson)
1019
+ } catch (err: TimeoutCancellationException) {
1020
+ GatewaySession.InvokeResult.error(
1021
+ code = "LOCATION_TIMEOUT",
1022
+ message = "LOCATION_TIMEOUT: no fix in time",
1023
+ )
1024
+ } catch (err: Throwable) {
1025
+ val message = err.message ?: "LOCATION_UNAVAILABLE: no fix"
1026
+ GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message)
1027
+ }
1028
+ }
1029
+ OpenClawScreenCommand.Record.rawValue -> {
1030
+ // Status pill mirrors screen recording state so it stays visible without overlay stacking.
1031
+ _screenRecordActive.value = true
1032
+ try {
1033
+ val res =
1034
+ try {
1035
+ screenRecorder.record(paramsJson)
1036
+ } catch (err: Throwable) {
1037
+ val (code, message) = invokeErrorFromThrowable(err)
1038
+ return GatewaySession.InvokeResult.error(code = code, message = message)
1039
+ }
1040
+ GatewaySession.InvokeResult.ok(res.payloadJson)
1041
+ } finally {
1042
+ _screenRecordActive.value = false
1043
+ }
1044
+ }
1045
+ OpenClawSmsCommand.Send.rawValue -> {
1046
+ val res = sms.send(paramsJson)
1047
+ if (res.ok) {
1048
+ GatewaySession.InvokeResult.ok(res.payloadJson)
1049
+ } else {
1050
+ val error = res.error ?: "SMS_SEND_FAILED"
1051
+ val idx = error.indexOf(':')
1052
+ val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED"
1053
+ GatewaySession.InvokeResult.error(code = code, message = error)
1054
+ }
1055
+ }
1056
+ else ->
1057
+ GatewaySession.InvokeResult.error(
1058
+ code = "INVALID_REQUEST",
1059
+ message = "INVALID_REQUEST: unknown command",
1060
+ )
1061
+ }
1062
+ }
1063
+
1064
+ private fun triggerCameraFlash() {
1065
+ // Token is used as a pulse trigger; value doesn't matter as long as it changes.
1066
+ _cameraFlashToken.value = SystemClock.elapsedRealtimeNanos()
1067
+ }
1068
+
1069
+ private fun showCameraHud(message: String, kind: CameraHudKind, autoHideMs: Long? = null) {
1070
+ val token = cameraHudSeq.incrementAndGet()
1071
+ _cameraHud.value = CameraHudState(token = token, kind = kind, message = message)
1072
+
1073
+ if (autoHideMs != null && autoHideMs > 0) {
1074
+ scope.launch {
1075
+ delay(autoHideMs)
1076
+ if (_cameraHud.value?.token == token) _cameraHud.value = null
1077
+ }
1078
+ }
1079
+ }
1080
+
1081
+ private fun invokeErrorFromThrowable(err: Throwable): Pair<String, String> {
1082
+ val raw = (err.message ?: "").trim()
1083
+ if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error"
1084
+
1085
+ val idx = raw.indexOf(':')
1086
+ if (idx <= 0) return "UNAVAILABLE" to raw
1087
+ val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" }
1088
+ val message = raw.substring(idx + 1).trim().ifEmpty { raw }
1089
+ // Preserve full string for callers/logging, but keep the returned message human-friendly.
1090
+ return code to "$code: $message"
1091
+ }
1092
+
1093
+ private fun parseLocationParams(paramsJson: String?): Triple<Long?, Long, String?> {
1094
+ if (paramsJson.isNullOrBlank()) {
1095
+ return Triple(null, 10_000L, null)
1096
+ }
1097
+ val root =
1098
+ try {
1099
+ json.parseToJsonElement(paramsJson).asObjectOrNull()
1100
+ } catch (_: Throwable) {
1101
+ null
1102
+ }
1103
+ val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull()
1104
+ val timeoutMs =
1105
+ (root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L)
1106
+ ?: 10_000L
1107
+ val desiredAccuracy =
1108
+ (root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase()
1109
+ return Triple(maxAgeMs, timeoutMs, desiredAccuracy)
1110
+ }
1111
+
1112
+ private fun resolveA2uiHostUrl(): String? {
1113
+ val nodeRaw = nodeSession.currentCanvasHostUrl()?.trim().orEmpty()
1114
+ val operatorRaw = operatorSession.currentCanvasHostUrl()?.trim().orEmpty()
1115
+ val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw
1116
+ if (raw.isBlank()) return null
1117
+ val base = raw.trimEnd('/')
1118
+ return "${base}/__openclaw__/a2ui/?platform=android"
1119
+ }
1120
+
1121
+ private suspend fun ensureA2uiReady(a2uiUrl: String): Boolean {
1122
+ try {
1123
+ val already = canvas.eval(a2uiReadyCheckJS)
1124
+ if (already == "true") return true
1125
+ } catch (_: Throwable) {
1126
+ // ignore
1127
+ }
1128
+
1129
+ canvas.navigate(a2uiUrl)
1130
+ repeat(50) {
1131
+ try {
1132
+ val ready = canvas.eval(a2uiReadyCheckJS)
1133
+ if (ready == "true") return true
1134
+ } catch (_: Throwable) {
1135
+ // ignore
1136
+ }
1137
+ delay(120)
1138
+ }
1139
+ return false
1140
+ }
1141
+
1142
+ private fun decodeA2uiMessages(command: String, paramsJson: String?): String {
1143
+ val raw = paramsJson?.trim().orEmpty()
1144
+ if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required")
1145
+
1146
+ val obj =
1147
+ json.parseToJsonElement(raw) as? JsonObject
1148
+ ?: throw IllegalArgumentException("INVALID_REQUEST: expected object params")
1149
+
1150
+ val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty()
1151
+ val hasMessagesArray = obj["messages"] is JsonArray
1152
+
1153
+ if (command == OpenClawCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) {
1154
+ val jsonl = jsonlField
1155
+ if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required")
1156
+ val messages =
1157
+ jsonl
1158
+ .lineSequence()
1159
+ .map { it.trim() }
1160
+ .filter { it.isNotBlank() }
1161
+ .mapIndexed { idx, line ->
1162
+ val el = json.parseToJsonElement(line)
1163
+ val msg =
1164
+ el as? JsonObject
1165
+ ?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object")
1166
+ validateA2uiV0_8(msg, idx + 1)
1167
+ msg
1168
+ }
1169
+ .toList()
1170
+ return JsonArray(messages).toString()
1171
+ }
1172
+
1173
+ val arr = obj["messages"] as? JsonArray ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required")
1174
+ val out =
1175
+ arr.mapIndexed { idx, el ->
1176
+ val msg =
1177
+ el as? JsonObject
1178
+ ?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object")
1179
+ validateA2uiV0_8(msg, idx + 1)
1180
+ msg
1181
+ }
1182
+ return JsonArray(out).toString()
1183
+ }
1184
+
1185
+ private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) {
1186
+ if (msg.containsKey("createSurface")) {
1187
+ throw IllegalArgumentException(
1188
+ "A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.",
1189
+ )
1190
+ }
1191
+ val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface")
1192
+ val matched = msg.keys.filter { allowed.contains(it) }
1193
+ if (matched.size != 1) {
1194
+ val found = msg.keys.sorted().joinToString(", ")
1195
+ throw IllegalArgumentException(
1196
+ "A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found",
1197
+ )
1198
+ }
1199
+ }
1200
+ }
1201
+
1202
+ private data class Quad<A, B, C, D>(val first: A, val second: B, val third: C, val fourth: D)
1203
+
1204
+ private const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A
1205
+
1206
+ private const val a2uiReadyCheckJS: String =
1207
+ """
1208
+ (() => {
1209
+ try {
1210
+ const host = globalThis.openclawA2UI;
1211
+ return !!host && typeof host.applyMessages === 'function';
1212
+ } catch (_) {
1213
+ return false;
1214
+ }
1215
+ })()
1216
+ """
1217
+
1218
+ private const val a2uiResetJS: String =
1219
+ """
1220
+ (() => {
1221
+ try {
1222
+ const host = globalThis.openclawA2UI;
1223
+ if (!host) return { ok: false, error: "missing openclawA2UI" };
1224
+ return host.reset();
1225
+ } catch (e) {
1226
+ return { ok: false, error: String(e?.message ?? e) };
1227
+ }
1228
+ })()
1229
+ """
1230
+
1231
+ private fun a2uiApplyMessagesJS(messagesJson: String): String {
1232
+ return """
1233
+ (() => {
1234
+ try {
1235
+ const host = globalThis.openclawA2UI;
1236
+ if (!host) return { ok: false, error: "missing openclawA2UI" };
1237
+ const messages = $messagesJson;
1238
+ return host.applyMessages(messages);
1239
+ } catch (e) {
1240
+ return { ok: false, error: String(e?.message ?? e) };
1241
+ }
1242
+ })()
1243
+ """.trimIndent()
1244
+ }
1245
+
1246
+ private fun String.toJsonString(): String {
1247
+ val escaped =
1248
+ this.replace("\\", "\\\\")
1249
+ .replace("\"", "\\\"")
1250
+ .replace("\n", "\\n")
1251
+ .replace("\r", "\\r")
1252
+ return "\"$escaped\""
1253
+ }
1254
+
1255
+ private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
1256
+
1257
+ private fun JsonElement?.asStringOrNull(): String? =
1258
+ when (this) {
1259
+ is JsonNull -> null
1260
+ is JsonPrimitive -> content
1261
+ else -> null
1262
+ }
1263
+
1264
+ private fun parseHexColorArgb(raw: String?): Long? {
1265
+ val trimmed = raw?.trim().orEmpty()
1266
+ if (trimmed.isEmpty()) return null
1267
+ val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed
1268
+ if (hex.length != 6) return null
1269
+ val rgb = hex.toLongOrNull(16) ?: return null
1270
+ return 0xFF000000L or rgb
1271
+ }
apps/android/app/src/main/java/ai/openclaw/android/PermissionRequester.kt ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package ai.openclaw.android
2
+
3
+ import android.content.pm.PackageManager
4
+ import android.content.Intent
5
+ import android.Manifest
6
+ import android.net.Uri
7
+ import android.provider.Settings
8
+ import androidx.appcompat.app.AlertDialog
9
+ import androidx.activity.ComponentActivity
10
+ import androidx.activity.result.ActivityResultLauncher
11
+ import androidx.activity.result.contract.ActivityResultContracts
12
+ import androidx.core.content.ContextCompat
13
+ import androidx.core.app.ActivityCompat
14
+ import kotlinx.coroutines.CompletableDeferred
15
+ import kotlinx.coroutines.Dispatchers
16
+ import kotlinx.coroutines.sync.Mutex
17
+ import kotlinx.coroutines.sync.withLock
18
+ import kotlinx.coroutines.withContext
19
+ import kotlinx.coroutines.suspendCancellableCoroutine
20
+ import kotlin.coroutines.resume
21
+
22
+ class PermissionRequester(private val activity: ComponentActivity) {
23
+ private val mutex = Mutex()
24
+ private var pending: CompletableDeferred<Map<String, Boolean>>? = null
25
+
26
+ private val launcher: ActivityResultLauncher<Array<String>> =
27
+ activity.registerForActivityResult(ActivityResultContracts.RequestMultiplePermissions()) { result ->
28
+ val p = pending
29
+ pending = null
30
+ p?.complete(result)
31
+ }
32
+
33
+ suspend fun requestIfMissing(
34
+ permissions: List<String>,
35
+ timeoutMs: Long = 20_000,
36
+ ): Map<String, Boolean> =
37
+ mutex.withLock {
38
+ val missing =
39
+ permissions.filter { perm ->
40
+ ContextCompat.checkSelfPermission(activity, perm) != PackageManager.PERMISSION_GRANTED
41
+ }
42
+ if (missing.isEmpty()) {
43
+ return permissions.associateWith { true }
44
+ }
45
+
46
+ val needsRationale =
47
+ missing.any { ActivityCompat.shouldShowRequestPermissionRationale(activity, it) }
48
+ if (needsRationale) {
49
+ val proceed = showRationaleDialog(missing)
50
+ if (!proceed) {
51
+ return permissions.associateWith { perm ->
52
+ ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
53
+ }
54
+ }
55
+ }
56
+
57
+ val deferred = CompletableDeferred<Map<String, Boolean>>()
58
+ pending = deferred
59
+ withContext(Dispatchers.Main) {
60
+ launcher.launch(missing.toTypedArray())
61
+ }
62
+
63
+ val result =
64
+ withContext(Dispatchers.Default) {
65
+ kotlinx.coroutines.withTimeout(timeoutMs) { deferred.await() }
66
+ }
67
+
68
+ // Merge: if something was already granted, treat it as granted even if launcher omitted it.
69
+ val merged =
70
+ permissions.associateWith { perm ->
71
+ val nowGranted =
72
+ ContextCompat.checkSelfPermission(activity, perm) == PackageManager.PERMISSION_GRANTED
73
+ result[perm] == true || nowGranted
74
+ }
75
+
76
+ val denied =
77
+ merged.filterValues { !it }.keys.filter {
78
+ !ActivityCompat.shouldShowRequestPermissionRationale(activity, it)
79
+ }
80
+ if (denied.isNotEmpty()) {
81
+ showSettingsDialog(denied)
82
+ }
83
+
84
+ return merged
85
+ }
86
+
87
+ private suspend fun showRationaleDialog(permissions: List<String>): Boolean =
88
+ withContext(Dispatchers.Main) {
89
+ suspendCancellableCoroutine { cont ->
90
+ AlertDialog.Builder(activity)
91
+ .setTitle("Permission required")
92
+ .setMessage(buildRationaleMessage(permissions))
93
+ .setPositiveButton("Continue") { _, _ -> cont.resume(true) }
94
+ .setNegativeButton("Not now") { _, _ -> cont.resume(false) }
95
+ .setOnCancelListener { cont.resume(false) }
96
+ .show()
97
+ }
98
+ }
99
+
100
+ private fun showSettingsDialog(permissions: List<String>) {
101
+ AlertDialog.Builder(activity)
102
+ .setTitle("Enable permission in Settings")
103
+ .setMessage(buildSettingsMessage(permissions))
104
+ .setPositiveButton("Open Settings") { _, _ ->
105
+ val intent =
106
+ Intent(
107
+ Settings.ACTION_APPLICATION_DETAILS_SETTINGS,
108
+ Uri.fromParts("package", activity.packageName, null),
109
+ )
110
+ activity.startActivity(intent)
111
+ }
112
+ .setNegativeButton("Cancel", null)
113
+ .show()
114
+ }
115
+
116
+ private fun buildRationaleMessage(permissions: List<String>): String {
117
+ val labels = permissions.map { permissionLabel(it) }
118
+ return "OpenClaw needs ${labels.joinToString(", ")} permissions to continue."
119
+ }
120
+
121
+ private fun buildSettingsMessage(permissions: List<String>): String {
122
+ val labels = permissions.map { permissionLabel(it) }
123
+ return "Please enable ${labels.joinToString(", ")} in Android Settings to continue."
124
+ }
125
+
126
+ private fun permissionLabel(permission: String): String =
127
+ when (permission) {
128
+ Manifest.permission.CAMERA -> "Camera"
129
+ Manifest.permission.RECORD_AUDIO -> "Microphone"
130
+ Manifest.permission.SEND_SMS -> "SMS"
131
+ else -> permission
132
+ }
133
+ }